2024-03-29T04:22:14.670 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-03-29T04:22:14.671 DEBUG:teuthology.run:Teuthology command: teuthology --description upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} --owner scheduled_teuthology@teuthology --name teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi --verbose --archive /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629121 -- /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629121/orig.config.yaml 2024-03-29T04:22:14.730 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-29T04:22:14.829 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629121 branch: squid description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' first_in_suite: false job_id: '7629121' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'librbd C and C++ api tests ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: squid sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 4408 sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 19/32 suite: upgrade suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba targets: smithi181.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIWTb+YgcVQFjXL09OeTN9p0qteWDa9xqwpXbDR6q9VTOOCcGqBY+i/8gpy2eVi44Oz4hh1r0/E/rXwvNwKwlFk= smithi187.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOHJtD/ZBPM72nmXa7NPgOaRpmzsm0gr+sOnHoP1jfpVVtEG400ZEHhZwb9mq4m1hbENWj882yRlaoJQfzDyA9k= tasks: - pexec: all: - sudo dnf install runc -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-03-29_02:08:11 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-03-29T04:22:14.829 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa; will attempt to use it 2024-03-29T04:22:14.830 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa/tasks 2024-03-29T04:22:14.830 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-03-29T04:22:14.831 INFO:teuthology.task.internal:Checking packages... 2024-03-29T04:22:14.856 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '8da23624bc17fe0b1f797227e72a0db0ce7821ba' 2024-03-29T04:22:14.856 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:22:14.856 INFO:teuthology.packaging:ref: None 2024-03-29T04:22:14.856 INFO:teuthology.packaging:tag: None 2024-03-29T04:22:14.856 INFO:teuthology.packaging:branch: squid 2024-03-29T04:22:14.856 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:22:14.857 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-03-29T04:22:15.106 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-1959.g8da23624 2024-03-29T04:22:15.108 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-03-29T04:22:15.115 INFO:teuthology.task.internal:no buildpackages task found 2024-03-29T04:22:15.115 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-03-29T04:22:15.135 INFO:teuthology.task.internal:Saving configuration 2024-03-29T04:22:15.156 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-03-29T04:22:15.164 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-29T04:22:15.189 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi181.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629121', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-29 04:17:20.549232', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIWTb+YgcVQFjXL09OeTN9p0qteWDa9xqwpXbDR6q9VTOOCcGqBY+i/8gpy2eVi44Oz4hh1r0/E/rXwvNwKwlFk='} 2024-03-29T04:22:15.214 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi187.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629121', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-29 04:17:20.550664', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOHJtD/ZBPM72nmXa7NPgOaRpmzsm0gr+sOnHoP1jfpVVtEG400ZEHhZwb9mq4m1hbENWj882yRlaoJQfzDyA9k='} 2024-03-29T04:22:15.215 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-03-29T04:22:15.223 INFO:teuthology.task.internal:roles: ubuntu@smithi181.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-03-29T04:22:15.224 INFO:teuthology.task.internal:roles: ubuntu@smithi187.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-03-29T04:22:15.224 INFO:teuthology.run_tasks:Running task console_log... 2024-03-29T04:22:15.294 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f140400d0d0>, signals=[15]) 2024-03-29T04:22:15.295 INFO:teuthology.run_tasks:Running task internal.connect... 2024-03-29T04:22:15.302 INFO:teuthology.task.internal:Opening connections... 2024-03-29T04:22:15.302 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi181.front.sepia.ceph.com 2024-03-29T04:22:15.304 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:22:15.434 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi187.front.sepia.ceph.com 2024-03-29T04:22:15.435 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi187.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:22:15.527 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-03-29T04:22:15.535 DEBUG:teuthology.orchestra.run.smithi181:> uname -m 2024-03-29T04:22:15.552 INFO:teuthology.orchestra.run.smithi181.stdout:x86_64 2024-03-29T04:22:15.552 DEBUG:teuthology.orchestra.run.smithi181:> cat /etc/os-release 2024-03-29T04:22:15.608 INFO:teuthology.orchestra.run.smithi181.stdout:NAME="CentOS Stream" 2024-03-29T04:22:15.609 INFO:teuthology.orchestra.run.smithi181.stdout:VERSION="9" 2024-03-29T04:22:15.609 INFO:teuthology.orchestra.run.smithi181.stdout:ID="centos" 2024-03-29T04:22:15.609 INFO:teuthology.orchestra.run.smithi181.stdout:ID_LIKE="rhel fedora" 2024-03-29T04:22:15.609 INFO:teuthology.orchestra.run.smithi181.stdout:VERSION_ID="9" 2024-03-29T04:22:15.609 INFO:teuthology.orchestra.run.smithi181.stdout:PLATFORM_ID="platform:el9" 2024-03-29T04:22:15.609 INFO:teuthology.orchestra.run.smithi181.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-29T04:22:15.609 INFO:teuthology.orchestra.run.smithi181.stdout:ANSI_COLOR="0;31" 2024-03-29T04:22:15.609 INFO:teuthology.orchestra.run.smithi181.stdout:LOGO="fedora-logo-icon" 2024-03-29T04:22:15.609 INFO:teuthology.orchestra.run.smithi181.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-29T04:22:15.609 INFO:teuthology.orchestra.run.smithi181.stdout:HOME_URL="https://centos.org/" 2024-03-29T04:22:15.609 INFO:teuthology.orchestra.run.smithi181.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-29T04:22:15.609 INFO:teuthology.orchestra.run.smithi181.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-29T04:22:15.610 INFO:teuthology.orchestra.run.smithi181.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-29T04:22:15.610 INFO:teuthology.lock.ops:Updating smithi181.front.sepia.ceph.com on lock server 2024-03-29T04:22:15.628 DEBUG:teuthology.orchestra.run.smithi187:> uname -m 2024-03-29T04:22:15.645 INFO:teuthology.orchestra.run.smithi187.stdout:x86_64 2024-03-29T04:22:15.645 DEBUG:teuthology.orchestra.run.smithi187:> cat /etc/os-release 2024-03-29T04:22:15.701 INFO:teuthology.orchestra.run.smithi187.stdout:NAME="CentOS Stream" 2024-03-29T04:22:15.701 INFO:teuthology.orchestra.run.smithi187.stdout:VERSION="9" 2024-03-29T04:22:15.702 INFO:teuthology.orchestra.run.smithi187.stdout:ID="centos" 2024-03-29T04:22:15.702 INFO:teuthology.orchestra.run.smithi187.stdout:ID_LIKE="rhel fedora" 2024-03-29T04:22:15.702 INFO:teuthology.orchestra.run.smithi187.stdout:VERSION_ID="9" 2024-03-29T04:22:15.702 INFO:teuthology.orchestra.run.smithi187.stdout:PLATFORM_ID="platform:el9" 2024-03-29T04:22:15.702 INFO:teuthology.orchestra.run.smithi187.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-29T04:22:15.702 INFO:teuthology.orchestra.run.smithi187.stdout:ANSI_COLOR="0;31" 2024-03-29T04:22:15.702 INFO:teuthology.orchestra.run.smithi187.stdout:LOGO="fedora-logo-icon" 2024-03-29T04:22:15.702 INFO:teuthology.orchestra.run.smithi187.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-29T04:22:15.702 INFO:teuthology.orchestra.run.smithi187.stdout:HOME_URL="https://centos.org/" 2024-03-29T04:22:15.702 INFO:teuthology.orchestra.run.smithi187.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-29T04:22:15.702 INFO:teuthology.orchestra.run.smithi187.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-29T04:22:15.702 INFO:teuthology.orchestra.run.smithi187.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-29T04:22:15.703 INFO:teuthology.lock.ops:Updating smithi187.front.sepia.ceph.com on lock server 2024-03-29T04:22:15.728 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-03-29T04:22:15.738 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-03-29T04:22:15.746 INFO:teuthology.task.internal:Checking for old test directory... 2024-03-29T04:22:15.746 DEBUG:teuthology.orchestra.run.smithi181:> test '!' -e /home/ubuntu/cephtest 2024-03-29T04:22:15.749 DEBUG:teuthology.orchestra.run.smithi187:> test '!' -e /home/ubuntu/cephtest 2024-03-29T04:22:15.764 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-03-29T04:22:15.771 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-03-29T04:22:15.773 DEBUG:teuthology.orchestra.run.smithi181:> test -z $(ls -A /var/lib/ceph) 2024-03-29T04:22:15.805 DEBUG:teuthology.orchestra.run.smithi187:> test -z $(ls -A /var/lib/ceph) 2024-03-29T04:22:15.908 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-03-29T04:22:15.966 INFO:teuthology.run_tasks:Running task kernel... 2024-03-29T04:22:15.982 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-03-29T04:22:15.982 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-03-29T04:22:15.982 DEBUG:teuthology.orchestra.run.smithi181:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-29T04:22:15.983 DEBUG:teuthology.orchestra.run.smithi187:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-29T04:22:15.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:22:15.999 DEBUG:teuthology.orchestra.run.smithi187:> uname -r 2024-03-29T04:22:16.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:22:16.000 DEBUG:teuthology.orchestra.run.smithi181:> uname -r 2024-03-29T04:22:16.056 INFO:teuthology.orchestra.run.smithi187.stdout:5.14.0-361.el9.x86_64 2024-03-29T04:22:16.056 INFO:teuthology.task.kernel:Running kernel on smithi187: 5.14.0-361.el9.x86_64 2024-03-29T04:22:16.056 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum install -y kernel 2024-03-29T04:22:16.058 INFO:teuthology.orchestra.run.smithi181.stdout:5.14.0-361.el9.x86_64 2024-03-29T04:22:16.059 INFO:teuthology.task.kernel:Running kernel on smithi181: 5.14.0-361.el9.x86_64 2024-03-29T04:22:16.059 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum install -y kernel 2024-03-29T04:22:18.115 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS-9 - AppStream 16 MB/s | 6.7 MB 00:00 2024-03-29T04:22:18.478 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS-9 - AppStream 9.3 MB/s | 6.7 MB 00:00 2024-03-29T04:22:19.699 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS-9 - BaseOS 4.1 MB/s | 2.0 MB 00:00 2024-03-29T04:22:20.112 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS-9 - BaseOS 3.7 MB/s | 2.0 MB 00:00 2024-03-29T04:22:22.302 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS Stream 9 - BaseOS 4.3 MB/s | 8.0 MB 00:01 2024-03-29T04:22:24.170 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS Stream 9 - AppStream 24 MB/s | 19 MB 00:00 2024-03-29T04:22:29.196 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS Stream 9 - BaseOS 898 kB/s | 8.0 MB 00:09 2024-03-29T04:22:30.370 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS Stream 9 - CRB 2.2 MB/s | 6.1 MB 00:02 2024-03-29T04:22:30.998 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS Stream 9 - AppStream 26 MB/s | 19 MB 00:00 2024-03-29T04:22:31.771 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS Stream 9 - Extras packages 57 kB/s | 16 kB 00:00 2024-03-29T04:22:33.450 INFO:teuthology.orchestra.run.smithi187.stdout:Extra Packages for Enterprise Linux 13 MB/s | 21 MB 00:01 2024-03-29T04:22:38.430 INFO:teuthology.orchestra.run.smithi187.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-03-29T04:22:40.720 INFO:teuthology.orchestra.run.smithi187.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-29T04:22:40.720 INFO:teuthology.orchestra.run.smithi187.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-29T04:22:40.820 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:22:40.825 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:22:40.825 INFO:teuthology.orchestra.run.smithi187.stdout: Package Arch Version Repository Size 2024-03-29T04:22:40.825 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:22:40.825 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:22:40.825 INFO:teuthology.orchestra.run.smithi187.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-03-29T04:22:40.825 INFO:teuthology.orchestra.run.smithi187.stdout:Installing dependencies: 2024-03-29T04:22:40.825 INFO:teuthology.orchestra.run.smithi187.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-03-29T04:22:40.825 INFO:teuthology.orchestra.run.smithi187.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-03-29T04:22:40.826 INFO:teuthology.orchestra.run.smithi187.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-03-29T04:22:40.826 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:22:40.826 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:22:40.826 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:22:40.826 INFO:teuthology.orchestra.run.smithi187.stdout:Install 4 Packages 2024-03-29T04:22:40.826 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:22:40.826 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 76 M 2024-03-29T04:22:40.827 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 123 M 2024-03-29T04:22:40.827 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:22:40.973 INFO:teuthology.orchestra.run.smithi187.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 1.3 MB/s | 192 kB 00:00 2024-03-29T04:22:41.648 INFO:teuthology.orchestra.run.smithi187.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 18 MB/s | 15 MB 00:00 2024-03-29T04:22:42.557 INFO:teuthology.orchestra.run.smithi187.stdout:(3/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 19 MB/s | 34 MB 00:01 2024-03-29T04:22:42.865 INFO:teuthology.orchestra.run.smithi187.stdout:(4/4): kernel-modules-core-5.14.0-434.el9.x86_6 15 MB/s | 28 MB 00:01 2024-03-29T04:22:42.868 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:22:42.869 INFO:teuthology.orchestra.run.smithi187.stdout:Total 37 MB/s | 76 MB 00:02 2024-03-29T04:22:42.869 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:22:43.235 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:22:43.235 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:22:43.463 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS Stream 9 - CRB 699 kB/s | 6.1 MB 00:08 2024-03-29T04:22:44.101 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:22:44.101 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:22:44.914 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS Stream 9 - Extras packages 49 kB/s | 16 kB 00:00 2024-03-29T04:22:46.361 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:22:46.951 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-03-29T04:22:47.009 INFO:teuthology.orchestra.run.smithi181.stdout:Extra Packages for Enterprise Linux 11 MB/s | 21 MB 00:01 2024-03-29T04:22:47.040 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:22:49.744 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:22:49.883 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:22:52.123 INFO:teuthology.orchestra.run.smithi181.stdout:lab-extras 6.7 kB/s | 1.7 kB 00:00 2024-03-29T04:22:54.397 INFO:teuthology.orchestra.run.smithi181.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-29T04:22:54.397 INFO:teuthology.orchestra.run.smithi181.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-29T04:22:54.485 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:22:54.490 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:22:54.490 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2024-03-29T04:22:54.490 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:22:54.490 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:22:54.490 INFO:teuthology.orchestra.run.smithi181.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-03-29T04:22:54.490 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2024-03-29T04:22:54.490 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-03-29T04:22:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-03-29T04:22:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-03-29T04:22:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:22:54.491 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:22:54.491 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:22:54.491 INFO:teuthology.orchestra.run.smithi181.stdout:Install 4 Packages 2024-03-29T04:22:54.491 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:22:54.492 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 76 M 2024-03-29T04:22:54.492 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 123 M 2024-03-29T04:22:54.492 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:22:54.595 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:22:54.641 INFO:teuthology.orchestra.run.smithi181.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 1.3 MB/s | 192 kB 00:00 2024-03-29T04:22:54.715 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:22:55.142 INFO:teuthology.orchestra.run.smithi181.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 23 MB/s | 15 MB 00:00 2024-03-29T04:22:55.717 INFO:teuthology.orchestra.run.smithi181.stdout:(3/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 27 MB/s | 34 MB 00:01 2024-03-29T04:22:56.059 INFO:teuthology.orchestra.run.smithi181.stdout:(4/4): kernel-modules-core-5.14.0-434.el9.x86_6 20 MB/s | 28 MB 00:01 2024-03-29T04:22:56.062 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:22:56.062 INFO:teuthology.orchestra.run.smithi181.stdout:Total 49 MB/s | 76 MB 00:01 2024-03-29T04:22:56.062 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:22:56.434 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:22:56.434 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:22:57.280 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:22:57.281 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:22:59.466 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:22:59.585 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:23:00.248 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-03-29T04:23:00.333 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:23:03.047 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:23:03.178 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:23:07.865 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:23:07.977 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:12.676 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:28.908 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:28.908 INFO:teuthology.orchestra.run.smithi187.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-29T04:23:28.908 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:23:28.953 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:29.292 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:29.292 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-03-29T04:23:29.292 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:23:29.292 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:23:29.684 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:29.684 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:23:29.684 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:23:29.684 INFO:teuthology.orchestra.run.smithi187.stdout: kernel-5.14.0-434.el9.x86_64 2024-03-29T04:23:29.684 INFO:teuthology.orchestra.run.smithi187.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-03-29T04:23:29.684 INFO:teuthology.orchestra.run.smithi187.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-03-29T04:23:29.684 INFO:teuthology.orchestra.run.smithi187.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-03-29T04:23:29.684 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:23:29.684 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:23:30.017 DEBUG:teuthology.orchestra.run.smithi187:> rpm -q kernel | sort -rV | head -n 1 2024-03-29T04:23:30.126 INFO:teuthology.orchestra.run.smithi187.stdout:kernel-5.14.0-434.el9.x86_64 2024-03-29T04:23:30.126 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-03-29T04:23:30.127 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-03-29T04:23:30.127 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-29T04:23:30.127 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-29T04:23:30.127 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-03-29T04:23:30.127 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi187.front.sepia.ceph.com, path=None, version=distro) 2024-03-29T04:23:30.127 DEBUG:teuthology.orchestra.run.smithi187:> rpm -q kernel | sort -rV | head -n 1 2024-03-29T04:23:30.207 INFO:teuthology.orchestra.run.smithi187.stdout:kernel-5.14.0-434.el9.x86_64 2024-03-29T04:23:30.208 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-03-29T04:23:30.208 DEBUG:teuthology.orchestra.run.smithi187:> sudo rpm -qi grub2-tools 2024-03-29T04:23:30.241 INFO:teuthology.orchestra.run.smithi187.stdout:Name : grub2-tools 2024-03-29T04:23:30.242 INFO:teuthology.orchestra.run.smithi187.stdout:Epoch : 1 2024-03-29T04:23:30.242 INFO:teuthology.orchestra.run.smithi187.stdout:Version : 2.06 2024-03-29T04:23:30.242 INFO:teuthology.orchestra.run.smithi187.stdout:Release : 68.el9 2024-03-29T04:23:30.242 INFO:teuthology.orchestra.run.smithi187.stdout:Architecture: x86_64 2024-03-29T04:23:30.242 INFO:teuthology.orchestra.run.smithi187.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-29T04:23:30.242 INFO:teuthology.orchestra.run.smithi187.stdout:Group : Unspecified 2024-03-29T04:23:30.242 INFO:teuthology.orchestra.run.smithi187.stdout:Size : 8264254 2024-03-29T04:23:30.242 INFO:teuthology.orchestra.run.smithi187.stdout:License : GPLv3+ 2024-03-29T04:23:30.242 INFO:teuthology.orchestra.run.smithi187.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-29T04:23:30.242 INFO:teuthology.orchestra.run.smithi187.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-29T04:23:30.242 INFO:teuthology.orchestra.run.smithi187.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-29T04:23:30.242 INFO:teuthology.orchestra.run.smithi187.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-29T04:23:30.242 INFO:teuthology.orchestra.run.smithi187.stdout:Packager : builder@centos.org 2024-03-29T04:23:30.243 INFO:teuthology.orchestra.run.smithi187.stdout:Vendor : CentOS 2024-03-29T04:23:30.243 INFO:teuthology.orchestra.run.smithi187.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-29T04:23:30.243 INFO:teuthology.orchestra.run.smithi187.stdout:Summary : Support tools for GRUB. 2024-03-29T04:23:30.243 INFO:teuthology.orchestra.run.smithi187.stdout:Description : 2024-03-29T04:23:30.243 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:23:30.243 INFO:teuthology.orchestra.run.smithi187.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-29T04:23:30.243 INFO:teuthology.orchestra.run.smithi187.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-29T04:23:30.243 INFO:teuthology.orchestra.run.smithi187.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-29T04:23:30.243 INFO:teuthology.orchestra.run.smithi187.stdout:hardware devices. 2024-03-29T04:23:30.243 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:23:30.243 INFO:teuthology.orchestra.run.smithi187.stdout:This subpackage provides tools for support of all platforms. 2024-03-29T04:23:30.244 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-29T04:23:30.244 INFO:teuthology.task.kernel:Updating grub on smithi187 to boot 5.14.0-434.el9.x86_64 2024-03-29T04:23:30.244 DEBUG:teuthology.orchestra.run.smithi187:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-29T04:23:30.850 INFO:teuthology.orchestra.run.smithi187.stderr:Generating grub configuration file ... 2024-03-29T04:23:32.239 INFO:teuthology.orchestra.run.smithi187.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-29T04:23:32.300 INFO:teuthology.orchestra.run.smithi187.stderr:done 2024-03-29T04:23:32.302 DEBUG:teuthology.orchestra.run.smithi187:> mktemp 2024-03-29T04:23:32.317 INFO:teuthology.orchestra.run.smithi187.stdout:/tmp/tmp.MP6w2EnAdW 2024-03-29T04:23:32.317 DEBUG:teuthology.orchestra.run.smithi187:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.MP6w2EnAdW 2024-03-29T04:23:32.394 DEBUG:teuthology.orchestra.run.smithi187:> sudo chmod 0666 /tmp/tmp.MP6w2EnAdW 2024-03-29T04:23:32.536 DEBUG:teuthology.orchestra.remote:smithi187:/tmp/tmp.MP6w2EnAdW is 6KB 2024-03-29T04:23:32.555 DEBUG:teuthology.orchestra.run.smithi187:> rm -fr /tmp/tmp.MP6w2EnAdW 2024-03-29T04:23:32.569 DEBUG:teuthology.orchestra.run.smithi187:> sudo /bin/ls /boot/loader/entries || true 2024-03-29T04:23:32.635 INFO:teuthology.orchestra.run.smithi187.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-29T04:23:32.635 INFO:teuthology.orchestra.run.smithi187.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-29T04:23:32.635 INFO:teuthology.orchestra.run.smithi187.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-29T04:23:32.635 INFO:teuthology.orchestra.run.smithi187.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-03-29T04:23:32.636 DEBUG:teuthology.orchestra.run.smithi187:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-03-29T04:23:32.789 DEBUG:teuthology.orchestra.run.smithi187:> sudo shutdown -r now 2024-03-29T04:23:41.971 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:41.972 INFO:teuthology.orchestra.run.smithi181.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-29T04:23:41.972 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:23:42.016 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:42.428 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:42.428 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-03-29T04:23:42.428 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:23:42.429 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:23:42.785 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:42.786 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:23:42.786 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:23:42.786 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-5.14.0-434.el9.x86_64 2024-03-29T04:23:42.786 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-03-29T04:23:42.786 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-03-29T04:23:42.786 INFO:teuthology.orchestra.run.smithi181.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-03-29T04:23:42.786 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:23:42.786 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:23:43.112 DEBUG:teuthology.orchestra.run.smithi181:> rpm -q kernel | sort -rV | head -n 1 2024-03-29T04:23:43.215 INFO:teuthology.orchestra.run.smithi181.stdout:kernel-5.14.0-434.el9.x86_64 2024-03-29T04:23:43.216 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-03-29T04:23:43.216 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-03-29T04:23:43.216 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-29T04:23:43.216 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-29T04:23:43.216 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-03-29T04:23:43.216 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi181.front.sepia.ceph.com, path=None, version=distro) 2024-03-29T04:23:43.216 DEBUG:teuthology.orchestra.run.smithi181:> rpm -q kernel | sort -rV | head -n 1 2024-03-29T04:23:43.296 INFO:teuthology.orchestra.run.smithi181.stdout:kernel-5.14.0-434.el9.x86_64 2024-03-29T04:23:43.297 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-03-29T04:23:43.297 DEBUG:teuthology.orchestra.run.smithi181:> sudo rpm -qi grub2-tools 2024-03-29T04:23:43.342 INFO:teuthology.orchestra.run.smithi181.stdout:Name : grub2-tools 2024-03-29T04:23:43.342 INFO:teuthology.orchestra.run.smithi181.stdout:Epoch : 1 2024-03-29T04:23:43.342 INFO:teuthology.orchestra.run.smithi181.stdout:Version : 2.06 2024-03-29T04:23:43.342 INFO:teuthology.orchestra.run.smithi181.stdout:Release : 68.el9 2024-03-29T04:23:43.342 INFO:teuthology.orchestra.run.smithi181.stdout:Architecture: x86_64 2024-03-29T04:23:43.342 INFO:teuthology.orchestra.run.smithi181.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-29T04:23:43.342 INFO:teuthology.orchestra.run.smithi181.stdout:Group : Unspecified 2024-03-29T04:23:43.343 INFO:teuthology.orchestra.run.smithi181.stdout:Size : 8264254 2024-03-29T04:23:43.343 INFO:teuthology.orchestra.run.smithi181.stdout:License : GPLv3+ 2024-03-29T04:23:43.343 INFO:teuthology.orchestra.run.smithi181.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-29T04:23:43.343 INFO:teuthology.orchestra.run.smithi181.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-29T04:23:43.343 INFO:teuthology.orchestra.run.smithi181.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-29T04:23:43.343 INFO:teuthology.orchestra.run.smithi181.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-29T04:23:43.343 INFO:teuthology.orchestra.run.smithi181.stdout:Packager : builder@centos.org 2024-03-29T04:23:43.343 INFO:teuthology.orchestra.run.smithi181.stdout:Vendor : CentOS 2024-03-29T04:23:43.343 INFO:teuthology.orchestra.run.smithi181.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-29T04:23:43.343 INFO:teuthology.orchestra.run.smithi181.stdout:Summary : Support tools for GRUB. 2024-03-29T04:23:43.343 INFO:teuthology.orchestra.run.smithi181.stdout:Description : 2024-03-29T04:23:43.343 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:23:43.343 INFO:teuthology.orchestra.run.smithi181.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-29T04:23:43.343 INFO:teuthology.orchestra.run.smithi181.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-29T04:23:43.343 INFO:teuthology.orchestra.run.smithi181.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-29T04:23:43.344 INFO:teuthology.orchestra.run.smithi181.stdout:hardware devices. 2024-03-29T04:23:43.344 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:23:43.344 INFO:teuthology.orchestra.run.smithi181.stdout:This subpackage provides tools for support of all platforms. 2024-03-29T04:23:43.345 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-29T04:23:43.345 INFO:teuthology.task.kernel:Updating grub on smithi181 to boot 5.14.0-434.el9.x86_64 2024-03-29T04:23:43.345 DEBUG:teuthology.orchestra.run.smithi181:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-29T04:23:43.912 INFO:teuthology.orchestra.run.smithi181.stderr:Generating grub configuration file ... 2024-03-29T04:23:45.271 INFO:teuthology.orchestra.run.smithi181.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-29T04:23:45.308 INFO:teuthology.orchestra.run.smithi181.stderr:done 2024-03-29T04:23:45.310 DEBUG:teuthology.orchestra.run.smithi181:> mktemp 2024-03-29T04:23:45.325 INFO:teuthology.orchestra.run.smithi181.stdout:/tmp/tmp.pJfTQ7XnNf 2024-03-29T04:23:45.326 DEBUG:teuthology.orchestra.run.smithi181:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.pJfTQ7XnNf 2024-03-29T04:23:45.394 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 0666 /tmp/tmp.pJfTQ7XnNf 2024-03-29T04:23:45.532 DEBUG:teuthology.orchestra.remote:smithi181:/tmp/tmp.pJfTQ7XnNf is 6KB 2024-03-29T04:23:45.552 DEBUG:teuthology.orchestra.run.smithi181:> rm -fr /tmp/tmp.pJfTQ7XnNf 2024-03-29T04:23:45.565 DEBUG:teuthology.orchestra.run.smithi181:> sudo /bin/ls /boot/loader/entries || true 2024-03-29T04:23:45.635 INFO:teuthology.orchestra.run.smithi181.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-29T04:23:45.635 INFO:teuthology.orchestra.run.smithi181.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-29T04:23:45.635 INFO:teuthology.orchestra.run.smithi181.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-29T04:23:45.635 INFO:teuthology.orchestra.run.smithi181.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-03-29T04:23:45.636 DEBUG:teuthology.orchestra.run.smithi181:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-03-29T04:23:45.783 DEBUG:teuthology.orchestra.run.smithi181:> sudo shutdown -r now 2024-03-29T04:24:02.806 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-03-29T04:24:02.807 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi187.front.sepia.ceph.com' 2024-03-29T04:24:02.808 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi187.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:24:15.798 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-03-29T04:24:15.799 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-03-29T04:24:15.799 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:24:21.278 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.187 2024-03-29T04:24:30.287 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi187.front.sepia.ceph.com' 2024-03-29T04:24:30.288 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi187.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:24:33.342 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.187 2024-03-29T04:24:34.334 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.181 2024-03-29T04:24:43.341 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-03-29T04:24:43.341 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:24:45.345 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi187.front.sepia.ceph.com' 2024-03-29T04:24:45.346 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi187.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:24:46.402 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.181 2024-03-29T04:24:58.414 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-03-29T04:24:58.415 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:25:17.122 DEBUG:teuthology.orchestra.run.smithi187:> true 2024-03-29T04:25:17.586 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi187.front.sepia.ceph.com' 2024-03-29T04:25:17.586 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-434.el9.x86_64"... 2024-03-29T04:25:17.587 DEBUG:teuthology.orchestra.run.smithi187:> uname -r 2024-03-29T04:25:17.600 INFO:teuthology.orchestra.run.smithi187.stdout:5.14.0-434.el9.x86_64 2024-03-29T04:25:17.600 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-03-29T04:25:17.600 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-29T04:25:17.601 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-29T04:25:18.601 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-03-29T04:25:18.602 DEBUG:teuthology.orchestra.run.smithi187:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-29T04:25:18.670 INFO:teuthology.orchestra.run.smithi187.stdout:ttyS1 2024-03-29T04:25:18.720 DEBUG:teuthology.parallel:result is None 2024-03-29T04:25:30.310 DEBUG:teuthology.orchestra.run.smithi181:> true 2024-03-29T04:25:30.758 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-03-29T04:25:30.758 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-434.el9.x86_64"... 2024-03-29T04:25:30.759 DEBUG:teuthology.orchestra.run.smithi181:> uname -r 2024-03-29T04:25:30.772 INFO:teuthology.orchestra.run.smithi181.stdout:5.14.0-434.el9.x86_64 2024-03-29T04:25:30.772 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-03-29T04:25:30.772 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-29T04:25:30.772 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-29T04:25:31.773 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-03-29T04:25:31.773 DEBUG:teuthology.orchestra.run.smithi181:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-29T04:25:31.864 INFO:teuthology.orchestra.run.smithi181.stdout:ttyS1 2024-03-29T04:25:31.901 DEBUG:teuthology.parallel:result is None 2024-03-29T04:25:31.901 INFO:teuthology.run_tasks:Running task internal.base... 2024-03-29T04:25:31.911 INFO:teuthology.task.internal:Creating test directory... 2024-03-29T04:25:31.912 DEBUG:teuthology.orchestra.run.smithi181:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-29T04:25:31.914 DEBUG:teuthology.orchestra.run.smithi187:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-29T04:25:31.935 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-03-29T04:25:31.944 INFO:teuthology.run_tasks:Running task internal.archive... 2024-03-29T04:25:31.952 INFO:teuthology.task.internal:Creating archive directory... 2024-03-29T04:25:31.952 DEBUG:teuthology.orchestra.run.smithi181:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-29T04:25:31.975 DEBUG:teuthology.orchestra.run.smithi187:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-29T04:25:32.285 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-03-29T04:25:32.291 INFO:teuthology.task.internal:Enabling coredump saving... 2024-03-29T04:25:32.292 DEBUG:teuthology.orchestra.run.smithi181:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-29T04:25:32.294 DEBUG:teuthology.orchestra.run.smithi187:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-29T04:25:32.335 INFO:teuthology.orchestra.run.smithi187.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-29T04:25:32.337 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-29T04:25:32.350 INFO:teuthology.orchestra.run.smithi187.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-29T04:25:32.351 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-29T04:25:32.353 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-03-29T04:25:32.360 INFO:teuthology.task.internal:Configuring sudo... 2024-03-29T04:25:32.360 DEBUG:teuthology.orchestra.run.smithi181:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-29T04:25:32.362 DEBUG:teuthology.orchestra.run.smithi187:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-29T04:25:32.422 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-03-29T04:25:32.431 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-03-29T04:25:32.431 DEBUG:teuthology.orchestra.run.smithi181:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-29T04:25:32.433 DEBUG:teuthology.orchestra.run.smithi187:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-29T04:25:32.478 DEBUG:teuthology.orchestra.run.smithi181:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-29T04:25:32.521 DEBUG:teuthology.orchestra.run.smithi181:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-29T04:25:32.752 DEBUG:teuthology.orchestra.run.smithi181:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-29T04:25:32.795 DEBUG:teuthology.orchestra.run.smithi181:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-29T04:25:32.837 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-03-29T04:25:32.837 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-29T04:25:32.902 DEBUG:teuthology.orchestra.run.smithi187:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-29T04:25:32.946 DEBUG:teuthology.orchestra.run.smithi187:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-29T04:25:33.004 DEBUG:teuthology.orchestra.run.smithi187:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-29T04:25:33.087 DEBUG:teuthology.orchestra.run.smithi187:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-29T04:25:33.133 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-29T04:25:33.133 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-29T04:25:33.203 DEBUG:teuthology.orchestra.run.smithi181:> sudo service rsyslog restart 2024-03-29T04:25:33.205 DEBUG:teuthology.orchestra.run.smithi187:> sudo service rsyslog restart 2024-03-29T04:25:33.255 INFO:teuthology.orchestra.run.smithi181.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-29T04:25:33.305 INFO:teuthology.orchestra.run.smithi187.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-29T04:25:33.634 INFO:teuthology.run_tasks:Running task internal.timer... 2024-03-29T04:25:33.642 INFO:teuthology.task.internal:Starting timer... 2024-03-29T04:25:33.642 INFO:teuthology.run_tasks:Running task pcp... 2024-03-29T04:25:33.666 INFO:teuthology.run_tasks:Running task selinux... 2024-03-29T04:25:33.676 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-03-29T04:25:33.677 DEBUG:teuthology.orchestra.run.smithi181:> sudo service auditd rotate 2024-03-29T04:25:33.740 INFO:teuthology.orchestra.run.smithi181.stdout:Rotating logs: 2024-03-29T04:25:33.741 DEBUG:teuthology.orchestra.run.smithi187:> sudo service auditd rotate 2024-03-29T04:25:33.791 INFO:teuthology.orchestra.run.smithi187.stdout:Rotating logs: 2024-03-29T04:25:33.793 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-03-29T04:25:33.793 DEBUG:teuthology.orchestra.run.smithi181:> /usr/sbin/getenforce 2024-03-29T04:25:33.815 INFO:teuthology.orchestra.run.smithi181.stdout:Permissive 2024-03-29T04:25:33.815 DEBUG:teuthology.orchestra.run.smithi187:> /usr/sbin/getenforce 2024-03-29T04:25:33.840 INFO:teuthology.orchestra.run.smithi187.stdout:Permissive 2024-03-29T04:25:33.840 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi181.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi187.front.sepia.ceph.com': 'permissive'} 2024-03-29T04:25:33.840 DEBUG:teuthology.orchestra.run.smithi181:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-03-29T04:25:33.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:25:33.879 DEBUG:teuthology.orchestra.run.smithi187:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-03-29T04:25:33.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:25:33.910 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-03-29T04:25:33.910 DEBUG:teuthology.orchestra.run.smithi181:> sudo /usr/sbin/setenforce permissive 2024-03-29T04:25:33.950 DEBUG:teuthology.orchestra.run.smithi187:> sudo /usr/sbin/setenforce permissive 2024-03-29T04:25:33.983 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-03-29T04:25:33.993 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-03-29T04:25:33.999 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-03-29T04:25:34.135 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-03-29T04:25:34.152 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-03-29T04:25:34.153 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi181.front.sepia.ceph.com,smithi187.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-03-29T04:34:04.827 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi181.front.sepia.ceph.com'), Remote(name='ubuntu@smithi187.front.sepia.ceph.com')] 2024-03-29T04:34:04.828 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-03-29T04:34:04.829 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi181.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:34:04.921 DEBUG:teuthology.orchestra.run.smithi181:> true 2024-03-29T04:34:04.996 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi181.front.sepia.ceph.com' 2024-03-29T04:34:04.997 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi187.front.sepia.ceph.com' 2024-03-29T04:34:04.997 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi187.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:34:05.086 DEBUG:teuthology.orchestra.run.smithi187:> true 2024-03-29T04:34:05.164 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi187.front.sepia.ceph.com' 2024-03-29T04:34:05.164 INFO:teuthology.run_tasks:Running task clock... 2024-03-29T04:34:05.178 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-03-29T04:34:05.178 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-29T04:34:05.179 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-29T04:34:05.181 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-29T04:34:05.182 DEBUG:teuthology.orchestra.run.smithi187:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-29T04:34:05.210 INFO:teuthology.orchestra.run.smithi181.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-29T04:34:05.227 INFO:teuthology.orchestra.run.smithi181.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-29T04:34:05.234 INFO:teuthology.orchestra.run.smithi187.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-29T04:34:05.250 INFO:teuthology.orchestra.run.smithi187.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-29T04:34:05.256 INFO:teuthology.orchestra.run.smithi181.stderr:sudo: ntpd: command not found 2024-03-29T04:34:05.270 INFO:teuthology.orchestra.run.smithi181.stdout:506 Cannot talk to daemon 2024-03-29T04:34:05.280 INFO:teuthology.orchestra.run.smithi187.stderr:sudo: ntpd: command not found 2024-03-29T04:34:05.287 INFO:teuthology.orchestra.run.smithi181.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-29T04:34:05.293 INFO:teuthology.orchestra.run.smithi187.stdout:506 Cannot talk to daemon 2024-03-29T04:34:05.303 INFO:teuthology.orchestra.run.smithi181.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-29T04:34:05.309 INFO:teuthology.orchestra.run.smithi187.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-29T04:34:05.326 INFO:teuthology.orchestra.run.smithi187.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-29T04:34:05.354 INFO:teuthology.orchestra.run.smithi181.stderr:bash: line 1: ntpq: command not found 2024-03-29T04:34:05.358 INFO:teuthology.orchestra.run.smithi181.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-29T04:34:05.358 INFO:teuthology.orchestra.run.smithi181.stdout:=============================================================================== 2024-03-29T04:34:05.358 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:34:05.358 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:34:05.358 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:34:05.358 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:34:05.375 INFO:teuthology.orchestra.run.smithi187.stderr:bash: line 1: ntpq: command not found 2024-03-29T04:34:05.379 INFO:teuthology.orchestra.run.smithi187.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-29T04:34:05.379 INFO:teuthology.orchestra.run.smithi187.stdout:=============================================================================== 2024-03-29T04:34:05.379 INFO:teuthology.orchestra.run.smithi187.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:34:05.379 INFO:teuthology.orchestra.run.smithi187.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:34:05.379 INFO:teuthology.orchestra.run.smithi187.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:34:05.379 INFO:teuthology.orchestra.run.smithi187.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:34:05.379 INFO:teuthology.run_tasks:Running task pexec... 2024-03-29T04:34:05.389 INFO:teuthology.task.pexec:Executing custom commands... 2024-03-29T04:34:05.389 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi181.front.sepia.ceph.com 2024-03-29T04:34:05.390 DEBUG:teuthology.orchestra.run.smithi181:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-03-29T04:34:05.390 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi187.front.sepia.ceph.com 2024-03-29T04:34:05.390 DEBUG:teuthology.orchestra.run.smithi187:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-03-29T04:34:05.986 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:00:46 ago on Fri 29 Mar 2024 04:33:19 AM UTC. 2024-03-29T04:34:06.010 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:05:05 ago on Fri 29 Mar 2024 04:29:00 AM UTC. 2024-03-29T04:34:06.134 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:34:06.135 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:34:06.135 INFO:teuthology.orchestra.run.smithi181.stdout: Package Architecture Version Repository Size 2024-03-29T04:34:06.136 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:34:06.136 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:34:06.136 INFO:teuthology.orchestra.run.smithi181.stdout: runc x86_64 4:1.1.12-2.el9 appstream 3.1 M 2024-03-29T04:34:06.136 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:34:06.136 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:34:06.136 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:34:06.136 INFO:teuthology.orchestra.run.smithi181.stdout:Install 1 Package 2024-03-29T04:34:06.136 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:34:06.136 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 3.1 M 2024-03-29T04:34:06.136 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 9.8 M 2024-03-29T04:34:06.136 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:34:06.156 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:34:06.158 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:34:06.158 INFO:teuthology.orchestra.run.smithi187.stdout: Package Architecture Version Repository Size 2024-03-29T04:34:06.158 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:34:06.158 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:34:06.158 INFO:teuthology.orchestra.run.smithi187.stdout: runc x86_64 4:1.1.12-2.el9 appstream 3.1 M 2024-03-29T04:34:06.158 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:34:06.158 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:34:06.159 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:34:06.159 INFO:teuthology.orchestra.run.smithi187.stdout:Install 1 Package 2024-03-29T04:34:06.159 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:34:06.159 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 3.1 M 2024-03-29T04:34:06.159 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 9.8 M 2024-03-29T04:34:06.159 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:34:06.788 INFO:teuthology.orchestra.run.smithi187.stdout:runc-1.1.12-2.el9.x86_64.rpm 5.3 MB/s | 3.1 MB 00:00 2024-03-29T04:34:06.788 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:34:06.788 INFO:teuthology.orchestra.run.smithi187.stdout:Total 5.0 MB/s | 3.1 MB 00:00 2024-03-29T04:34:06.825 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:34:06.831 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:34:06.831 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:34:06.910 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:34:06.911 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:34:07.120 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:34:07.139 INFO:teuthology.orchestra.run.smithi181.stdout:runc-1.1.12-2.el9.x86_64.rpm 3.5 MB/s | 3.1 MB 00:00 2024-03-29T04:34:07.139 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:34:07.140 INFO:teuthology.orchestra.run.smithi181.stdout:Total 3.1 MB/s | 3.1 MB 00:01 2024-03-29T04:34:07.176 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:34:07.181 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:34:07.181 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:34:07.207 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : runc-4:1.1.12-2.el9.x86_64 1/1 2024-03-29T04:34:07.261 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:34:07.261 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:34:07.471 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:34:07.551 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : runc-4:1.1.12-2.el9.x86_64 1/1 2024-03-29T04:34:07.782 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: runc-4:1.1.12-2.el9.x86_64 1/1 2024-03-29T04:34:08.069 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: runc-4:1.1.12-2.el9.x86_64 1/1 2024-03-29T04:34:08.096 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : runc-4:1.1.12-2.el9.x86_64 1/1 2024-03-29T04:34:08.097 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:34:08.097 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:34:08.097 INFO:teuthology.orchestra.run.smithi187.stdout: runc-4:1.1.12-2.el9.x86_64 2024-03-29T04:34:08.097 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:34:08.097 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:34:08.334 DEBUG:teuthology.parallel:result is None 2024-03-29T04:34:08.365 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : runc-4:1.1.12-2.el9.x86_64 1/1 2024-03-29T04:34:08.365 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:34:08.365 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:34:08.365 INFO:teuthology.orchestra.run.smithi181.stdout: runc-4:1.1.12-2.el9.x86_64 2024-03-29T04:34:08.365 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:34:08.365 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:34:08.649 DEBUG:teuthology.parallel:result is None 2024-03-29T04:34:08.649 INFO:teuthology.run_tasks:Running task install... 2024-03-29T04:34:08.658 DEBUG:teuthology.task.install:project ceph 2024-03-29T04:34:08.658 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '8da23624bc17fe0b1f797227e72a0db0ce7821ba'}} 2024-03-29T04:34:08.658 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '8da23624bc17fe0b1f797227e72a0db0ce7821ba'} 2024-03-29T04:34:08.658 INFO:teuthology.task.install:Using flavor: default 2024-03-29T04:34:08.664 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-03-29T04:34:08.664 INFO:teuthology.task.install:extra packages: [] 2024-03-29T04:34:08.665 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '8da23624bc17fe0b1f797227e72a0db0ce7821ba', 'tag': None, 'wait_for_package': False} 2024-03-29T04:34:08.665 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:34:08.665 INFO:teuthology.packaging:ref: None 2024-03-29T04:34:08.665 INFO:teuthology.packaging:tag: None 2024-03-29T04:34:08.665 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:34:08.665 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:34:08.666 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-03-29T04:34:08.667 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '8da23624bc17fe0b1f797227e72a0db0ce7821ba', 'tag': None, 'wait_for_package': False} 2024-03-29T04:34:08.668 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:34:08.668 INFO:teuthology.packaging:ref: None 2024-03-29T04:34:08.668 INFO:teuthology.packaging:tag: None 2024-03-29T04:34:08.668 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:34:08.668 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:34:08.668 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-03-29T04:34:08.980 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/ 2024-03-29T04:34:08.981 INFO:teuthology.task.install.rpm:Package version is 17.2.7-767.g9b4b8ffd 2024-03-29T04:34:08.984 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/ 2024-03-29T04:34:08.985 INFO:teuthology.task.install.rpm:Package version is 17.2.7-767.g9b4b8ffd 2024-03-29T04:34:09.096 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-03-29T04:34:09.096 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-03-29T04:34:09.096 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-03-29T04:34:09.123 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-03-29T04:34:09.123 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:34:09.124 INFO:teuthology.packaging:ref: None 2024-03-29T04:34:09.124 INFO:teuthology.packaging:tag: None 2024-03-29T04:34:09.124 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:34:09.124 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:34:09.124 DEBUG:teuthology.orchestra.run.smithi181:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-03-29T04:34:09.150 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-03-29T04:34:09.150 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-29T04:34:09.150 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-03-29T04:34:09.179 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-03-29T04:34:09.179 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:34:09.180 INFO:teuthology.packaging:ref: None 2024-03-29T04:34:09.180 INFO:teuthology.packaging:tag: None 2024-03-29T04:34:09.180 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:34:09.180 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:34:09.180 DEBUG:teuthology.orchestra.run.smithi187:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-03-29T04:34:09.191 DEBUG:teuthology.orchestra.run.smithi181:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-03-29T04:34:09.248 DEBUG:teuthology.orchestra.run.smithi187:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-03-29T04:34:09.282 DEBUG:teuthology.orchestra.run.smithi181:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-03-29T04:34:09.317 INFO:teuthology.orchestra.run.smithi181.stdout:check_obsoletes = 1 2024-03-29T04:34:09.318 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum clean all 2024-03-29T04:34:09.328 DEBUG:teuthology.orchestra.run.smithi187:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-03-29T04:34:09.394 INFO:teuthology.orchestra.run.smithi187.stdout:check_obsoletes = 1 2024-03-29T04:34:09.396 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum clean all 2024-03-29T04:34:09.559 INFO:teuthology.orchestra.run.smithi181.stdout:74 files removed 2024-03-29T04:34:09.586 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-radosgw 2024-03-29T04:34:09.648 INFO:teuthology.orchestra.run.smithi187.stdout:74 files removed 2024-03-29T04:34:09.675 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install ceph-radosgw 2024-03-29T04:34:10.246 INFO:teuthology.orchestra.run.smithi181.stdout:ceph packages for x86_64 218 kB/s | 77 kB 00:00 2024-03-29T04:34:10.344 INFO:teuthology.orchestra.run.smithi187.stdout:ceph packages for x86_64 211 kB/s | 77 kB 00:00 2024-03-29T04:34:10.613 INFO:teuthology.orchestra.run.smithi181.stdout:ceph noarch packages 29 kB/s | 9.1 kB 00:00 2024-03-29T04:34:10.769 INFO:teuthology.orchestra.run.smithi187.stdout:ceph noarch packages 24 kB/s | 9.1 kB 00:00 2024-03-29T04:34:10.997 INFO:teuthology.orchestra.run.smithi181.stdout:ceph source packages 5.1 kB/s | 1.8 kB 00:00 2024-03-29T04:34:11.095 INFO:teuthology.orchestra.run.smithi187.stdout:ceph source packages 6.2 kB/s | 1.8 kB 00:00 2024-03-29T04:34:11.424 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS-9 - AppStream 17 MB/s | 6.7 MB 00:00 2024-03-29T04:34:11.747 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS-9 - AppStream 11 MB/s | 6.7 MB 00:00 2024-03-29T04:34:12.808 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS-9 - BaseOS 7.2 MB/s | 2.0 MB 00:00 2024-03-29T04:34:13.158 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS-9 - BaseOS 6.5 MB/s | 2.0 MB 00:00 2024-03-29T04:34:14.132 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS Stream 9 - BaseOS 8.0 MB/s | 8.0 MB 00:00 2024-03-29T04:34:14.707 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS Stream 9 - BaseOS 6.6 MB/s | 8.0 MB 00:01 2024-03-29T04:34:16.934 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS Stream 9 - AppStream 11 MB/s | 19 MB 00:01 2024-03-29T04:34:16.975 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS Stream 9 - AppStream 16 MB/s | 19 MB 00:01 2024-03-29T04:34:20.884 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS Stream 9 - CRB 15 MB/s | 6.1 MB 00:00 2024-03-29T04:34:21.918 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS Stream 9 - CRB 4.1 MB/s | 6.1 MB 00:01 2024-03-29T04:34:22.510 INFO:teuthology.orchestra.run.smithi187.stdout:CentOS Stream 9 - Extras packages 33 kB/s | 16 kB 00:00 2024-03-29T04:34:23.377 INFO:teuthology.orchestra.run.smithi181.stdout:CentOS Stream 9 - Extras packages 53 kB/s | 16 kB 00:00 2024-03-29T04:34:23.895 INFO:teuthology.orchestra.run.smithi187.stdout:Extra Packages for Enterprise Linux 16 MB/s | 21 MB 00:01 2024-03-29T04:34:25.447 INFO:teuthology.orchestra.run.smithi181.stdout:Extra Packages for Enterprise Linux 11 MB/s | 21 MB 00:01 2024-03-29T04:34:28.845 INFO:teuthology.orchestra.run.smithi187.stdout:lab-extras 34 kB/s | 1.7 kB 00:00 2024-03-29T04:34:30.386 INFO:teuthology.orchestra.run.smithi181.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-03-29T04:34:31.444 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:34:31.446 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:34:31.446 INFO:teuthology.orchestra.run.smithi187.stdout: Package Arch Version Repository Size 2024-03-29T04:34:31.446 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:34:31.446 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:34:31.446 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-radosgw x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 14 M 2024-03-29T04:34:31.446 INFO:teuthology.orchestra.run.smithi187.stdout:Upgrading: 2024-03-29T04:34:31.446 INFO:teuthology.orchestra.run.smithi187.stdout: librados2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 3.1 M 2024-03-29T04:34:31.446 INFO:teuthology.orchestra.run.smithi187.stdout: librbd1 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.9 M 2024-03-29T04:34:31.446 INFO:teuthology.orchestra.run.smithi187.stdout:Installing dependencies: 2024-03-29T04:34:31.446 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-base x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 5.0 M 2024-03-29T04:34:31.446 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-common x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 22 M 2024-03-29T04:34:31.447 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-selinux x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 25 k 2024-03-29T04:34:31.447 INFO:teuthology.orchestra.run.smithi187.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-03-29T04:34:31.447 INFO:teuthology.orchestra.run.smithi187.stdout: libcephfs2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 636 k 2024-03-29T04:34:31.447 INFO:teuthology.orchestra.run.smithi187.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-03-29T04:34:31.447 INFO:teuthology.orchestra.run.smithi187.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-03-29T04:34:31.447 INFO:teuthology.orchestra.run.smithi187.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-03-29T04:34:31.447 INFO:teuthology.orchestra.run.smithi187.stdout: libradosstriper1 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 458 k 2024-03-29T04:34:31.447 INFO:teuthology.orchestra.run.smithi187.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-03-29T04:34:31.447 INFO:teuthology.orchestra.run.smithi187.stdout: librgw2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 4.9 M 2024-03-29T04:34:31.447 INFO:teuthology.orchestra.run.smithi187.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-03-29T04:34:31.447 INFO:teuthology.orchestra.run.smithi187.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-03-29T04:34:31.447 INFO:teuthology.orchestra.run.smithi187.stdout: python3-ceph-argparse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 45 k 2024-03-29T04:34:31.447 INFO:teuthology.orchestra.run.smithi187.stdout: python3-ceph-common x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 118 k 2024-03-29T04:34:31.447 INFO:teuthology.orchestra.run.smithi187.stdout: python3-cephfs x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 156 k 2024-03-29T04:34:31.448 INFO:teuthology.orchestra.run.smithi187.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-03-29T04:34:31.448 INFO:teuthology.orchestra.run.smithi187.stdout: python3-rados x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 319 k 2024-03-29T04:34:31.448 INFO:teuthology.orchestra.run.smithi187.stdout: python3-rbd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 293 k 2024-03-29T04:34:31.448 INFO:teuthology.orchestra.run.smithi187.stdout: python3-rgw x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 99 k 2024-03-29T04:34:31.448 INFO:teuthology.orchestra.run.smithi187.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-03-29T04:34:31.448 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:34:31.448 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:34:31.448 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:34:31.448 INFO:teuthology.orchestra.run.smithi187.stdout:Install 22 Packages 2024-03-29T04:34:31.448 INFO:teuthology.orchestra.run.smithi187.stdout:Upgrade 2 Packages 2024-03-29T04:34:31.448 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:34:31.449 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 58 M 2024-03-29T04:34:31.449 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:34:32.986 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:34:32.987 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:34:32.987 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2024-03-29T04:34:32.987 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:34:32.987 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:34:32.988 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-radosgw x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 14 M 2024-03-29T04:34:32.988 INFO:teuthology.orchestra.run.smithi181.stdout:Upgrading: 2024-03-29T04:34:32.988 INFO:teuthology.orchestra.run.smithi181.stdout: librados2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 3.1 M 2024-03-29T04:34:32.988 INFO:teuthology.orchestra.run.smithi181.stdout: librbd1 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.9 M 2024-03-29T04:34:32.988 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2024-03-29T04:34:32.988 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-base x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 5.0 M 2024-03-29T04:34:32.988 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-common x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 22 M 2024-03-29T04:34:32.988 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-selinux x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 25 k 2024-03-29T04:34:32.988 INFO:teuthology.orchestra.run.smithi181.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-03-29T04:34:32.988 INFO:teuthology.orchestra.run.smithi181.stdout: libcephfs2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 636 k 2024-03-29T04:34:32.988 INFO:teuthology.orchestra.run.smithi181.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-03-29T04:34:32.988 INFO:teuthology.orchestra.run.smithi181.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-03-29T04:34:32.988 INFO:teuthology.orchestra.run.smithi181.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-03-29T04:34:32.988 INFO:teuthology.orchestra.run.smithi181.stdout: libradosstriper1 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 458 k 2024-03-29T04:34:32.989 INFO:teuthology.orchestra.run.smithi181.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-03-29T04:34:32.989 INFO:teuthology.orchestra.run.smithi181.stdout: librgw2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 4.9 M 2024-03-29T04:34:32.989 INFO:teuthology.orchestra.run.smithi181.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-03-29T04:34:32.989 INFO:teuthology.orchestra.run.smithi181.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-03-29T04:34:32.989 INFO:teuthology.orchestra.run.smithi181.stdout: python3-ceph-argparse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 45 k 2024-03-29T04:34:32.989 INFO:teuthology.orchestra.run.smithi181.stdout: python3-ceph-common x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 118 k 2024-03-29T04:34:32.989 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cephfs x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 156 k 2024-03-29T04:34:32.989 INFO:teuthology.orchestra.run.smithi181.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-03-29T04:34:32.989 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rados x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 319 k 2024-03-29T04:34:32.989 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rbd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 293 k 2024-03-29T04:34:32.989 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rgw x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 99 k 2024-03-29T04:34:32.989 INFO:teuthology.orchestra.run.smithi181.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-03-29T04:34:32.989 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:34:32.989 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:34:32.990 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:34:32.990 INFO:teuthology.orchestra.run.smithi181.stdout:Install 22 Packages 2024-03-29T04:34:32.990 INFO:teuthology.orchestra.run.smithi181.stdout:Upgrade 2 Packages 2024-03-29T04:34:32.990 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:34:32.990 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 58 M 2024-03-29T04:34:32.990 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:34:33.355 INFO:teuthology.orchestra.run.smithi187.stdout:(1/24): ceph-radosgw-17.2.7-767.g9b4b8ffd.el9.x 9.5 MB/s | 14 MB 00:01 2024-03-29T04:34:33.396 INFO:teuthology.orchestra.run.smithi187.stdout:(2/24): ceph-selinux-17.2.7-767.g9b4b8ffd.el9.x 607 kB/s | 25 kB 00:00 2024-03-29T04:34:33.497 INFO:teuthology.orchestra.run.smithi187.stdout:(3/24): libcephfs2-17.2.7-767.g9b4b8ffd.el9.x86 6.2 MB/s | 636 kB 00:00 2024-03-29T04:34:33.580 INFO:teuthology.orchestra.run.smithi187.stdout:(4/24): libradosstriper1-17.2.7-767.g9b4b8ffd.e 5.4 MB/s | 458 kB 00:00 2024-03-29T04:34:33.939 INFO:teuthology.orchestra.run.smithi187.stdout:(5/24): ceph-common-17.2.7-767.g9b4b8ffd.el9.x8 11 MB/s | 22 MB 00:02 2024-03-29T04:34:33.981 INFO:teuthology.orchestra.run.smithi187.stdout:(6/24): python3-ceph-argparse-17.2.7-767.g9b4b8 1.1 MB/s | 45 kB 00:00 2024-03-29T04:34:34.048 INFO:teuthology.orchestra.run.smithi187.stdout:(7/24): librgw2-17.2.7-767.g9b4b8ffd.el9.x86_64 10 MB/s | 4.9 MB 00:00 2024-03-29T04:34:34.081 INFO:teuthology.orchestra.run.smithi187.stdout:(8/24): python3-ceph-common-17.2.7-767.g9b4b8ff 1.1 MB/s | 118 kB 00:00 2024-03-29T04:34:34.107 INFO:teuthology.orchestra.run.smithi187.stdout:(9/24): python3-cephfs-17.2.7-767.g9b4b8ffd.el9 2.6 MB/s | 156 kB 00:00 2024-03-29T04:34:34.157 INFO:teuthology.orchestra.run.smithi187.stdout:(10/24): python3-rados-17.2.7-767.g9b4b8ffd.el9 4.1 MB/s | 319 kB 00:00 2024-03-29T04:34:34.182 INFO:teuthology.orchestra.run.smithi187.stdout:(11/24): python3-rbd-17.2.7-767.g9b4b8ffd.el9.x 3.8 MB/s | 293 kB 00:00 2024-03-29T04:34:34.207 INFO:teuthology.orchestra.run.smithi187.stdout:(12/24): python3-rgw-17.2.7-767.g9b4b8ffd.el9.x 1.9 MB/s | 99 kB 00:00 2024-03-29T04:34:34.391 INFO:teuthology.orchestra.run.smithi187.stdout:(13/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 241 kB/s | 44 kB 00:00 2024-03-29T04:34:34.433 INFO:teuthology.orchestra.run.smithi187.stdout:(14/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 635 kB/s | 159 kB 00:00 2024-03-29T04:34:34.525 INFO:teuthology.orchestra.run.smithi187.stdout:(15/24): librdkafka-1.6.1-102.el9.x86_64.rpm 4.8 MB/s | 661 kB 00:00 2024-03-29T04:34:34.550 INFO:teuthology.orchestra.run.smithi187.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.4 MB/s | 282 kB 00:00 2024-03-29T04:34:34.567 INFO:teuthology.orchestra.run.smithi187.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 989 kB/s | 41 kB 00:00 2024-03-29T04:34:34.784 INFO:teuthology.orchestra.run.smithi187.stdout:(18/24): liboath-2.6.7-2.el9.x86_64.rpm 224 kB/s | 49 kB 00:00 2024-03-29T04:34:34.909 INFO:teuthology.orchestra.run.smithi187.stdout:(19/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 861 kB/s | 309 kB 00:00 2024-03-29T04:34:34.929 INFO:teuthology.orchestra.run.smithi181.stdout:(1/24): ceph-radosgw-17.2.7-767.g9b4b8ffd.el9.x 7.6 MB/s | 14 MB 00:01 2024-03-29T04:34:34.943 INFO:teuthology.orchestra.run.smithi187.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 425 kB/s | 67 kB 00:00 2024-03-29T04:34:34.980 INFO:teuthology.orchestra.run.smithi181.stdout:(2/24): ceph-selinux-17.2.7-767.g9b4b8ffd.el9.x 505 kB/s | 25 kB 00:00 2024-03-29T04:34:35.072 INFO:teuthology.orchestra.run.smithi181.stdout:(3/24): libcephfs2-17.2.7-767.g9b4b8ffd.el9.x86 6.8 MB/s | 636 kB 00:00 2024-03-29T04:34:35.139 INFO:teuthology.orchestra.run.smithi181.stdout:(4/24): libradosstriper1-17.2.7-767.g9b4b8ffd.e 6.7 MB/s | 458 kB 00:00 2024-03-29T04:34:35.143 INFO:teuthology.orchestra.run.smithi187.stdout:(21/24): thrift-0.15.0-2.el9.x86_64.rpm 6.8 MB/s | 1.6 MB 00:00 2024-03-29T04:34:35.339 INFO:teuthology.orchestra.run.smithi181.stdout:(5/24): ceph-base-17.2.7-767.g9b4b8ffd.el9.x86_ 2.2 MB/s | 5.0 MB 00:02 2024-03-29T04:34:35.427 INFO:teuthology.orchestra.run.smithi187.stdout:(22/24): librados2-17.2.7-767.g9b4b8ffd.el9.x86 6.4 MB/s | 3.1 MB 00:00 2024-03-29T04:34:35.431 INFO:teuthology.orchestra.run.smithi181.stdout:(6/24): python3-ceph-argparse-17.2.7-767.g9b4b8 493 kB/s | 45 kB 00:00 2024-03-29T04:34:35.523 INFO:teuthology.orchestra.run.smithi181.stdout:(7/24): python3-ceph-common-17.2.7-767.g9b4b8ff 1.3 MB/s | 118 kB 00:00 2024-03-29T04:34:35.615 INFO:teuthology.orchestra.run.smithi181.stdout:(8/24): python3-cephfs-17.2.7-767.g9b4b8ffd.el9 1.7 MB/s | 156 kB 00:00 2024-03-29T04:34:35.627 INFO:teuthology.orchestra.run.smithi187.stdout:(23/24): librbd1-17.2.7-767.g9b4b8ffd.el9.x86_6 6.1 MB/s | 2.9 MB 00:00 2024-03-29T04:34:35.707 INFO:teuthology.orchestra.run.smithi181.stdout:(9/24): librgw2-17.2.7-767.g9b4b8ffd.el9.x86_64 8.6 MB/s | 4.9 MB 00:00 2024-03-29T04:34:35.732 INFO:teuthology.orchestra.run.smithi181.stdout:(10/24): python3-rados-17.2.7-767.g9b4b8ffd.el9 2.7 MB/s | 319 kB 00:00 2024-03-29T04:34:35.766 INFO:teuthology.orchestra.run.smithi181.stdout:(11/24): python3-rbd-17.2.7-767.g9b4b8ffd.el9.x 4.9 MB/s | 293 kB 00:00 2024-03-29T04:34:35.799 INFO:teuthology.orchestra.run.smithi181.stdout:(12/24): python3-rgw-17.2.7-767.g9b4b8ffd.el9.x 1.5 MB/s | 99 kB 00:00 2024-03-29T04:34:35.949 INFO:teuthology.orchestra.run.smithi181.stdout:(13/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 360 kB/s | 44 kB 00:00 2024-03-29T04:34:35.975 INFO:teuthology.orchestra.run.smithi181.stdout:(14/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 762 kB/s | 159 kB 00:00 2024-03-29T04:34:36.036 INFO:teuthology.orchestra.run.smithi187.stdout:(24/24): ceph-base-17.2.7-767.g9b4b8ffd.el9.x86 1.2 MB/s | 5.0 MB 00:04 2024-03-29T04:34:36.041 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:34:36.041 INFO:teuthology.orchestra.run.smithi187.stdout:Total 13 MB/s | 58 MB 00:04 2024-03-29T04:34:36.041 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:34:36.075 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:34:36.075 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:34:36.083 INFO:teuthology.orchestra.run.smithi181.stdout:(15/24): librdkafka-1.6.1-102.el9.x86_64.rpm 4.8 MB/s | 661 kB 00:00 2024-03-29T04:34:36.109 INFO:teuthology.orchestra.run.smithi181.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.1 MB/s | 282 kB 00:00 2024-03-29T04:34:36.125 INFO:teuthology.orchestra.run.smithi181.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 994 kB/s | 41 kB 00:00 2024-03-29T04:34:36.539 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:34:36.540 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:34:37.293 INFO:teuthology.orchestra.run.smithi181.stdout:(18/24): ceph-common-17.2.7-767.g9b4b8ffd.el9.x 5.3 MB/s | 22 MB 00:04 2024-03-29T04:34:37.725 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:34:38.050 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-03-29T04:34:38.113 INFO:teuthology.orchestra.run.smithi187.stdout: Upgrading : librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:34:38.209 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:34:38.293 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/26 2024-03-29T04:34:38.323 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:34:38.411 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:34:38.464 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-03-29T04:34:38.494 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-03-29T04:34:38.924 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-03-29T04:34:38.951 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 8/26 2024-03-29T04:34:39.013 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 8/26 2024-03-29T04:34:39.042 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-03-29T04:34:39.085 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x 10/26 2024-03-29T04:34:39.122 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 11/26 2024-03-29T04:34:39.185 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 12/26 2024-03-29T04:34:39.212 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:34:39.278 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:34:39.318 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-03-29T04:34:39.354 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-03-29T04:34:39.387 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-03-29T04:34:39.680 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-03-29T04:34:39.708 INFO:teuthology.orchestra.run.smithi187.stdout: Upgrading : librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 18/26 2024-03-29T04:34:39.789 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 18/26 2024-03-29T04:34:39.862 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 19/26 2024-03-29T04:34:39.885 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86 20/26 2024-03-29T04:34:42.336 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:34:42.374 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:34:42.886 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:34:42.919 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 22/26 2024-03-29T04:34:43.021 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 22/26 2024-03-29T04:34:43.021 INFO:teuthology.orchestra.run.smithi187.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-03-29T04:34:43.021 INFO:teuthology.orchestra.run.smithi187.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-03-29T04:34:43.021 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:34:43.062 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:34:43.167 INFO:teuthology.orchestra.run.smithi181.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 6.9 kB/s | 49 kB 00:07 2024-03-29T04:34:52.641 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:34:52.641 INFO:teuthology.orchestra.run.smithi187.stdout:skipping the directory /sys 2024-03-29T04:34:52.641 INFO:teuthology.orchestra.run.smithi187.stdout:skipping the directory /proc 2024-03-29T04:34:52.641 INFO:teuthology.orchestra.run.smithi187.stdout:skipping the directory /mnt 2024-03-29T04:34:52.641 INFO:teuthology.orchestra.run.smithi187.stdout:skipping the directory /var/tmp 2024-03-29T04:34:52.641 INFO:teuthology.orchestra.run.smithi187.stdout:skipping the directory /home 2024-03-29T04:34:52.641 INFO:teuthology.orchestra.run.smithi187.stdout:skipping the directory /root 2024-03-29T04:34:52.641 INFO:teuthology.orchestra.run.smithi187.stdout:skipping the directory /tmp 2024-03-29T04:34:52.641 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:34:53.926 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 24/26 2024-03-29T04:34:54.004 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 24/26 2024-03-29T04:34:54.004 INFO:teuthology.orchestra.run.smithi187.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:34:54.004 INFO:teuthology.orchestra.run.smithi187.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-03-29T04:34:54.004 INFO:teuthology.orchestra.run.smithi187.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-29T04:34:54.004 INFO:teuthology.orchestra.run.smithi187.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-29T04:34:54.004 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:34:54.006 INFO:teuthology.orchestra.run.smithi187.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-03-29T04:34:54.087 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-03-29T04:34:54.087 INFO:teuthology.orchestra.run.smithi187.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:34:55.300 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:34:55.300 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/26 2024-03-29T04:34:55.300 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:34:55.300 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/26 2024-03-29T04:34:55.300 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:34:55.300 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/26 2024-03-29T04:34:55.300 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/26 2024-03-29T04:34:55.301 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 7/26 2024-03-29T04:34:55.301 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x 8/26 2024-03-29T04:34:55.301 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86 9/26 2024-03-29T04:34:55.301 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 10/26 2024-03-29T04:34:55.301 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 11/26 2024-03-29T04:34:55.301 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 12/26 2024-03-29T04:34:55.301 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:34:55.301 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-03-29T04:34:55.301 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-03-29T04:34:55.301 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-03-29T04:34:55.301 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-03-29T04:34:55.301 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-03-29T04:34:55.301 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-03-29T04:34:55.301 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-03-29T04:34:55.301 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-03-29T04:34:55.301 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-03-29T04:34:55.302 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:34:55.302 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-03-29T04:34:55.302 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 25/26 2024-03-29T04:34:56.275 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:34:56.275 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:34:56.275 INFO:teuthology.orchestra.run.smithi187.stdout:Upgraded: 2024-03-29T04:34:56.275 INFO:teuthology.orchestra.run.smithi187.stdout: librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:34:56.275 INFO:teuthology.orchestra.run.smithi187.stdout: librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:34:56.276 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:34:56.276 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:34:56.276 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:34:56.276 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:34:56.276 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:34:56.276 INFO:teuthology.orchestra.run.smithi187.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-03-29T04:34:56.276 INFO:teuthology.orchestra.run.smithi187.stdout: libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:34:56.276 INFO:teuthology.orchestra.run.smithi187.stdout: liboath-2.6.7-2.el9.x86_64 2024-03-29T04:34:56.276 INFO:teuthology.orchestra.run.smithi187.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-03-29T04:34:56.276 INFO:teuthology.orchestra.run.smithi187.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-03-29T04:34:56.276 INFO:teuthology.orchestra.run.smithi187.stdout: libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:34:56.276 INFO:teuthology.orchestra.run.smithi187.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-03-29T04:34:56.276 INFO:teuthology.orchestra.run.smithi187.stdout: librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:34:56.276 INFO:teuthology.orchestra.run.smithi187.stdout: libunwind-1.6.2-1.el9.x86_64 2024-03-29T04:34:56.277 INFO:teuthology.orchestra.run.smithi187.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-03-29T04:34:56.277 INFO:teuthology.orchestra.run.smithi187.stdout: python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:34:56.277 INFO:teuthology.orchestra.run.smithi187.stdout: python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:34:56.277 INFO:teuthology.orchestra.run.smithi187.stdout: python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:34:56.277 INFO:teuthology.orchestra.run.smithi187.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-03-29T04:34:56.277 INFO:teuthology.orchestra.run.smithi187.stdout: python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:34:56.277 INFO:teuthology.orchestra.run.smithi187.stdout: python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:34:56.277 INFO:teuthology.orchestra.run.smithi187.stdout: python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:34:56.277 INFO:teuthology.orchestra.run.smithi187.stdout: thrift-0.15.0-2.el9.x86_64 2024-03-29T04:34:56.277 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:34:56.277 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:34:56.587 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install ceph-test 2024-03-29T04:34:57.154 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:00:29 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:34:57.311 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:34:57.311 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:34:57.311 INFO:teuthology.orchestra.run.smithi187.stdout: Package Arch Version Repository Size 2024-03-29T04:34:57.311 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:34:57.312 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:34:57.312 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-test x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 48 M 2024-03-29T04:34:57.312 INFO:teuthology.orchestra.run.smithi187.stdout:Installing dependencies: 2024-03-29T04:34:57.312 INFO:teuthology.orchestra.run.smithi187.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-03-29T04:34:57.312 INFO:teuthology.orchestra.run.smithi187.stdout: libcephsqlite x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 163 k 2024-03-29T04:34:57.312 INFO:teuthology.orchestra.run.smithi187.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-AppStream 217 k 2024-03-29T04:34:57.312 INFO:teuthology.orchestra.run.smithi187.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-03-29T04:34:57.312 INFO:teuthology.orchestra.run.smithi187.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-03-29T04:34:57.312 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:34:57.312 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:34:57.312 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:34:57.312 INFO:teuthology.orchestra.run.smithi187.stdout:Install 6 Packages 2024-03-29T04:34:57.312 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:34:57.313 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 49 M 2024-03-29T04:34:57.313 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 193 M 2024-03-29T04:34:57.313 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:34:57.541 INFO:teuthology.orchestra.run.smithi187.stdout:(1/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 1.3 MB/s | 217 kB 00:00 2024-03-29T04:34:57.583 INFO:teuthology.orchestra.run.smithi187.stdout:(2/6): socat-1.7.4.1-5.el9.x86_64.rpm 7.1 MB/s | 300 kB 00:00 2024-03-29T04:34:57.617 INFO:teuthology.orchestra.run.smithi187.stdout:(3/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.8 MB/s | 63 kB 00:00 2024-03-29T04:34:57.792 INFO:teuthology.orchestra.run.smithi187.stdout:(4/6): libcephsqlite-17.2.7-767.g9b4b8ffd.el9.x 390 kB/s | 163 kB 00:00 2024-03-29T04:34:57.817 INFO:teuthology.orchestra.run.smithi187.stdout:(5/6): jq-1.6-16.el9.x86_64.rpm 935 kB/s | 187 kB 00:00 2024-03-29T04:35:00.268 INFO:teuthology.orchestra.run.smithi187.stdout:(6/6): ceph-test-17.2.7-767.g9b4b8ffd.el9.x86_6 17 MB/s | 48 MB 00:02 2024-03-29T04:35:00.271 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:35:00.271 INFO:teuthology.orchestra.run.smithi187.stdout:Total 16 MB/s | 49 MB 00:02 2024-03-29T04:35:00.285 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:35:00.305 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:35:00.305 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:35:00.708 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:35:00.708 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:35:01.275 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:35:01.390 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-03-29T04:35:01.428 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-03-29T04:35:01.461 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-03-29T04:35:01.501 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-03-29T04:35:01.529 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/6 2024-03-29T04:35:05.915 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/6 2024-03-29T04:35:05.996 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/6 2024-03-29T04:35:08.274 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/6 2024-03-29T04:35:08.274 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/6 2024-03-29T04:35:08.274 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/6 2024-03-29T04:35:08.274 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-03-29T04:35:08.275 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 4/6 2024-03-29T04:35:08.275 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/6 2024-03-29T04:35:08.636 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : jq-1.6-16.el9.x86_64 6/6 2024-03-29T04:35:08.636 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:08.636 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:35:08.636 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:35:08.636 INFO:teuthology.orchestra.run.smithi187.stdout: jq-1.6-16.el9.x86_64 2024-03-29T04:35:08.636 INFO:teuthology.orchestra.run.smithi187.stdout: libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:35:08.636 INFO:teuthology.orchestra.run.smithi187.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-03-29T04:35:08.636 INFO:teuthology.orchestra.run.smithi187.stdout: socat-1.7.4.1-5.el9.x86_64 2024-03-29T04:35:08.637 INFO:teuthology.orchestra.run.smithi187.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-03-29T04:35:08.637 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:08.637 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:35:08.918 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install ceph 2024-03-29T04:35:09.483 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:00:41 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:35:09.636 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:35:09.639 INFO:teuthology.orchestra.run.smithi187.stdout:======================================================================================== 2024-03-29T04:35:09.639 INFO:teuthology.orchestra.run.smithi187.stdout: Package Arch Version Repository Size 2024-03-29T04:35:09.639 INFO:teuthology.orchestra.run.smithi187.stdout:======================================================================================== 2024-03-29T04:35:09.639 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:35:09.639 INFO:teuthology.orchestra.run.smithi187.stdout: ceph x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 6.5 k 2024-03-29T04:35:09.639 INFO:teuthology.orchestra.run.smithi187.stdout:Installing dependencies: 2024-03-29T04:35:09.639 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-mds x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.1 M 2024-03-29T04:35:09.639 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-mgr x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 1.4 M 2024-03-29T04:35:09.639 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-mgr-modules-core noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 240 k 2024-03-29T04:35:09.640 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-mon x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 3.9 M 2024-03-29T04:35:09.640 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-osd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 15 M 2024-03-29T04:35:09.640 INFO:teuthology.orchestra.run.smithi187.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-03-29T04:35:09.640 INFO:teuthology.orchestra.run.smithi187.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-03-29T04:35:09.640 INFO:teuthology.orchestra.run.smithi187.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-03-29T04:35:09.640 INFO:teuthology.orchestra.run.smithi187.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-03-29T04:35:09.640 INFO:teuthology.orchestra.run.smithi187.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-03-29T04:35:09.640 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-03-29T04:35:09.640 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-03-29T04:35:09.640 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-03-29T04:35:09.640 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-03-29T04:35:09.640 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-03-29T04:35:09.641 INFO:teuthology.orchestra.run.smithi187.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-03-29T04:35:09.641 INFO:teuthology.orchestra.run.smithi187.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-03-29T04:35:09.641 INFO:teuthology.orchestra.run.smithi187.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-03-29T04:35:09.641 INFO:teuthology.orchestra.run.smithi187.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-03-29T04:35:09.641 INFO:teuthology.orchestra.run.smithi187.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-03-29T04:35:09.641 INFO:teuthology.orchestra.run.smithi187.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-03-29T04:35:09.641 INFO:teuthology.orchestra.run.smithi187.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-03-29T04:35:09.641 INFO:teuthology.orchestra.run.smithi187.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-03-29T04:35:09.641 INFO:teuthology.orchestra.run.smithi187.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-03-29T04:35:09.641 INFO:teuthology.orchestra.run.smithi187.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-03-29T04:35:09.641 INFO:teuthology.orchestra.run.smithi187.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-03-29T04:35:09.641 INFO:teuthology.orchestra.run.smithi187.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-03-29T04:35:09.641 INFO:teuthology.orchestra.run.smithi187.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-03-29T04:35:09.641 INFO:teuthology.orchestra.run.smithi187.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-03-29T04:35:09.642 INFO:teuthology.orchestra.run.smithi187.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-03-29T04:35:09.642 INFO:teuthology.orchestra.run.smithi187.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-03-29T04:35:09.642 INFO:teuthology.orchestra.run.smithi187.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-03-29T04:35:09.642 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:09.642 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:35:09.642 INFO:teuthology.orchestra.run.smithi187.stdout:======================================================================================== 2024-03-29T04:35:09.642 INFO:teuthology.orchestra.run.smithi187.stdout:Install 33 Packages 2024-03-29T04:35:09.642 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:09.643 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 27 M 2024-03-29T04:35:09.643 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 96 M 2024-03-29T04:35:09.643 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:35:09.954 INFO:teuthology.orchestra.run.smithi187.stdout:(1/33): ceph-17.2.7-767.g9b4b8ffd.el9.x86_64.rp 33 kB/s | 6.5 kB 00:00 2024-03-29T04:35:10.296 INFO:teuthology.orchestra.run.smithi187.stdout:(2/33): ceph-mgr-17.2.7-767.g9b4b8ffd.el9.x86_6 2.6 MB/s | 1.4 MB 00:00 2024-03-29T04:35:10.805 INFO:teuthology.orchestra.run.smithi187.stdout:(3/33): ceph-mds-17.2.7-767.g9b4b8ffd.el9.x86_6 2.0 MB/s | 2.1 MB 00:01 2024-03-29T04:35:10.914 INFO:teuthology.orchestra.run.smithi187.stdout:(4/33): ceph-mgr-modules-core-17.2.7-767.g9b4b8 2.2 MB/s | 240 kB 00:00 2024-03-29T04:35:11.031 INFO:teuthology.orchestra.run.smithi187.stdout:(5/33): python3-mako-1.1.4-6.el9.noarch.rpm 1.2 MB/s | 144 kB 00:00 2024-03-29T04:35:11.056 INFO:teuthology.orchestra.run.smithi187.stdout:(6/33): python3-markupsafe-1.1.1-12.el9.x86_64. 1.2 MB/s | 32 kB 00:00 2024-03-29T04:35:11.090 INFO:teuthology.orchestra.run.smithi187.stdout:(7/33): python3-pytz-2021.1-5.el9.noarch.rpm 1.4 MB/s | 47 kB 00:00 2024-03-29T04:35:11.115 INFO:teuthology.orchestra.run.smithi187.stdout:(8/33): python3-toml-0.10.2-6.el9.noarch.rpm 1.5 MB/s | 39 kB 00:00 2024-03-29T04:35:11.173 INFO:teuthology.orchestra.run.smithi187.stdout:(9/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 4.0 MB/s | 241 kB 00:00 2024-03-29T04:35:11.240 INFO:teuthology.orchestra.run.smithi187.stdout:(10/33): python3-cryptography-36.0.1-4.el9.x86_ 17 MB/s | 1.1 MB 00:00 2024-03-29T04:35:11.299 INFO:teuthology.orchestra.run.smithi187.stdout:(11/33): ceph-mon-17.2.7-767.g9b4b8ffd.el9.x86_ 2.9 MB/s | 3.9 MB 00:01 2024-03-29T04:35:11.316 INFO:teuthology.orchestra.run.smithi187.stdout:(12/33): python3-ply-3.11-14.el9.noarch.rpm 1.3 MB/s | 103 kB 00:00 2024-03-29T04:35:11.333 INFO:teuthology.orchestra.run.smithi187.stdout:(13/33): python3-pycparser-2.20-6.el9.noarch.rp 3.7 MB/s | 124 kB 00:00 2024-03-29T04:35:11.358 INFO:teuthology.orchestra.run.smithi187.stdout:(14/33): python3-urllib3-1.26.5-5.el9.noarch.rp 4.4 MB/s | 187 kB 00:00 2024-03-29T04:35:11.525 INFO:teuthology.orchestra.run.smithi187.stdout:(15/33): ceph-osd-17.2.7-767.g9b4b8ffd.el9.x86_ 12 MB/s | 15 MB 00:01 2024-03-29T04:35:11.742 INFO:teuthology.orchestra.run.smithi187.stdout:(16/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 113 kB/s | 43 kB 00:00 2024-03-29T04:35:11.917 INFO:teuthology.orchestra.run.smithi187.stdout:(17/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 468 kB/s | 172 kB 00:00 2024-03-29T04:35:11.943 INFO:teuthology.orchestra.run.smithi187.stdout:(18/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.7 MB/s | 358 kB 00:00 2024-03-29T04:35:12.010 INFO:teuthology.orchestra.run.smithi187.stdout:(19/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 116 kB/s | 11 kB 00:00 2024-03-29T04:35:12.035 INFO:teuthology.orchestra.run.smithi187.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 192 kB/s | 18 kB 00:00 2024-03-29T04:35:12.068 INFO:teuthology.orchestra.run.smithi187.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 394 kB/s | 23 kB 00:00 2024-03-29T04:35:12.094 INFO:teuthology.orchestra.run.smithi187.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 331 kB/s | 19 kB 00:00 2024-03-29T04:35:12.128 INFO:teuthology.orchestra.run.smithi187.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 334 kB/s | 20 kB 00:00 2024-03-29T04:35:12.153 INFO:teuthology.orchestra.run.smithi187.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 788 kB/s | 46 kB 00:00 2024-03-29T04:35:12.203 INFO:teuthology.orchestra.run.smithi187.stdout:(25/33): python3-more-itertools-8.12.0-2.el9.no 1.0 MB/s | 79 kB 00:00 2024-03-29T04:35:12.228 INFO:teuthology.orchestra.run.smithi187.stdout:(26/33): python3-requests-2.25.1-8.el9.noarch.r 139 kB/s | 125 kB 00:00 2024-03-29T04:35:12.270 INFO:teuthology.orchestra.run.smithi187.stdout:(27/33): python3-pecan-1.4.2-3.el9.noarch.rpm 2.3 MB/s | 272 kB 00:00 2024-03-29T04:35:12.287 INFO:teuthology.orchestra.run.smithi187.stdout:(28/33): python3-portend-3.1.0-2.el9.noarch.rpm 197 kB/s | 16 kB 00:00 2024-03-29T04:35:12.312 INFO:teuthology.orchestra.run.smithi187.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.1 MB/s | 90 kB 00:00 2024-03-29T04:35:12.338 INFO:teuthology.orchestra.run.smithi187.stdout:(30/33): python3-tempora-5.0.0-2.el9.noarch.rpm 532 kB/s | 36 kB 00:00 2024-03-29T04:35:12.380 INFO:teuthology.orchestra.run.smithi187.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 2.4 MB/s | 230 kB 00:00 2024-03-29T04:35:12.413 INFO:teuthology.orchestra.run.smithi187.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 265 kB/s | 20 kB 00:00 2024-03-29T04:35:12.438 INFO:teuthology.orchestra.run.smithi187.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.3 MB/s | 427 kB 00:00 2024-03-29T04:35:12.443 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:35:12.443 INFO:teuthology.orchestra.run.smithi187.stdout:Total 9.6 MB/s | 27 MB 00:02 2024-03-29T04:35:12.457 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:35:12.493 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:35:12.493 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:35:12.829 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:35:12.830 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:35:13.399 INFO:teuthology.orchestra.run.smithi181.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 1.9 kB/s | 67 kB 00:36 2024-03-29T04:35:13.991 INFO:teuthology.orchestra.run.smithi181.stdout:(21/24): librados2-17.2.7-767.g9b4b8ffd.el9.x86 5.2 MB/s | 3.1 MB 00:00 2024-03-29T04:35:14.114 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:35:14.177 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-03-29T04:35:14.209 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-03-29T04:35:14.250 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-03-29T04:35:14.333 INFO:teuthology.orchestra.run.smithi181.stdout:(22/24): librbd1-17.2.7-767.g9b4b8ffd.el9.x86_6 8.6 MB/s | 2.9 MB 00:00 2024-03-29T04:35:14.341 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-03-29T04:35:14.420 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-03-29T04:35:14.462 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-03-29T04:35:14.533 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-03-29T04:35:14.595 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 8/33 2024-03-29T04:35:14.633 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 9/33 2024-03-29T04:35:14.682 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-ply-3.11-14.el9.noarch 10/33 2024-03-29T04:35:14.739 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 11/33 2024-03-29T04:35:14.923 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 12/33 2024-03-29T04:35:14.971 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 13/33 2024-03-29T04:35:15.021 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 14/33 2024-03-29T04:35:15.061 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 15/33 2024-03-29T04:35:15.091 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 16/33 2024-03-29T04:35:15.167 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 17/33 2024-03-29T04:35:15.201 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-03-29T04:35:15.226 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 19/33 2024-03-29T04:35:15.258 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 20/33 2024-03-29T04:35:15.293 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 21/33 2024-03-29T04:35:15.327 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 22/33 2024-03-29T04:35:15.417 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 23/33 2024-03-29T04:35:15.459 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 24/33 2024-03-29T04:35:15.530 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 25/33 2024-03-29T04:35:15.608 INFO:teuthology.orchestra.run.smithi181.stdout:(23/24): thrift-0.15.0-2.el9.x86_64.rpm 50 kB/s | 1.6 MB 00:32 2024-03-29T04:35:15.668 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 26/33 2024-03-29T04:35:15.794 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-03-29T04:35:15.940 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.n 28/33 2024-03-29T04:35:15.972 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 29/33 2024-03-29T04:35:15.998 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 29/33 2024-03-29T04:35:15.998 INFO:teuthology.orchestra.run.smithi187.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:35:15.998 INFO:teuthology.orchestra.run.smithi187.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-03-29T04:35:15.998 INFO:teuthology.orchestra.run.smithi187.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-29T04:35:15.999 INFO:teuthology.orchestra.run.smithi187.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-29T04:35:15.999 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:17.298 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 30/33 2024-03-29T04:35:17.324 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 30/33 2024-03-29T04:35:17.324 INFO:teuthology.orchestra.run.smithi187.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:35:17.325 INFO:teuthology.orchestra.run.smithi187.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-03-29T04:35:17.325 INFO:teuthology.orchestra.run.smithi187.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-29T04:35:17.325 INFO:teuthology.orchestra.run.smithi187.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-29T04:35:17.325 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:17.698 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 31/33 2024-03-29T04:35:17.721 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 31/33 2024-03-29T04:35:17.721 INFO:teuthology.orchestra.run.smithi187.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:35:17.721 INFO:teuthology.orchestra.run.smithi187.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-03-29T04:35:17.721 INFO:teuthology.orchestra.run.smithi187.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-29T04:35:17.721 INFO:teuthology.orchestra.run.smithi187.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-29T04:35:17.722 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:17.924 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 32/33 2024-03-29T04:35:17.947 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 32/33 2024-03-29T04:35:17.947 INFO:teuthology.orchestra.run.smithi187.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:35:17.947 INFO:teuthology.orchestra.run.smithi187.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-03-29T04:35:17.947 INFO:teuthology.orchestra.run.smithi187.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-29T04:35:17.947 INFO:teuthology.orchestra.run.smithi187.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-29T04:35:17.947 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:18.021 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 33/33 2024-03-29T04:35:19.703 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 33/33 2024-03-29T04:35:19.703 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/33 2024-03-29T04:35:19.703 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/33 2024-03-29T04:35:19.703 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/33 2024-03-29T04:35:19.703 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/33 2024-03-29T04:35:19.703 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/33 2024-03-29T04:35:19.703 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.n 6/33 2024-03-29T04:35:19.703 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 7/33 2024-03-29T04:35:19.704 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 8/33 2024-03-29T04:35:19.704 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 9/33 2024-03-29T04:35:19.704 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 10/33 2024-03-29T04:35:19.704 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 11/33 2024-03-29T04:35:19.704 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 12/33 2024-03-29T04:35:19.704 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-ply-3.11-14.el9.noarch 13/33 2024-03-29T04:35:19.704 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 14/33 2024-03-29T04:35:19.704 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 15/33 2024-03-29T04:35:19.704 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 16/33 2024-03-29T04:35:19.704 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-03-29T04:35:19.704 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-03-29T04:35:19.704 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-03-29T04:35:19.704 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-03-29T04:35:19.705 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-03-29T04:35:19.705 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-03-29T04:35:19.705 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-03-29T04:35:19.705 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-03-29T04:35:19.705 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-03-29T04:35:19.705 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-03-29T04:35:19.705 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-03-29T04:35:19.705 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-03-29T04:35:19.706 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-03-29T04:35:19.706 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-03-29T04:35:19.706 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-03-29T04:35:19.706 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-03-29T04:35:20.615 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-03-29T04:35:20.615 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:20.616 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:35:20.616 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:35:20.616 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:35:20.616 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:35:20.616 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:35:20.616 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:35:20.616 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:35:20.616 INFO:teuthology.orchestra.run.smithi187.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-03-29T04:35:20.616 INFO:teuthology.orchestra.run.smithi187.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-03-29T04:35:20.616 INFO:teuthology.orchestra.run.smithi187.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-03-29T04:35:20.616 INFO:teuthology.orchestra.run.smithi187.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-03-29T04:35:20.616 INFO:teuthology.orchestra.run.smithi187.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-03-29T04:35:20.616 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-03-29T04:35:20.616 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-03-29T04:35:20.616 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-03-29T04:35:20.617 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-03-29T04:35:20.617 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-03-29T04:35:20.617 INFO:teuthology.orchestra.run.smithi187.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-03-29T04:35:20.617 INFO:teuthology.orchestra.run.smithi187.stdout: python3-mako-1.1.4-6.el9.noarch 2024-03-29T04:35:20.617 INFO:teuthology.orchestra.run.smithi187.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-03-29T04:35:20.617 INFO:teuthology.orchestra.run.smithi187.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-03-29T04:35:20.617 INFO:teuthology.orchestra.run.smithi187.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-03-29T04:35:20.617 INFO:teuthology.orchestra.run.smithi187.stdout: python3-ply-3.11-14.el9.noarch 2024-03-29T04:35:20.617 INFO:teuthology.orchestra.run.smithi187.stdout: python3-portend-3.1.0-2.el9.noarch 2024-03-29T04:35:20.617 INFO:teuthology.orchestra.run.smithi187.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-03-29T04:35:20.617 INFO:teuthology.orchestra.run.smithi187.stdout: python3-pycparser-2.20-6.el9.noarch 2024-03-29T04:35:20.617 INFO:teuthology.orchestra.run.smithi187.stdout: python3-pytz-2021.1-5.el9.noarch 2024-03-29T04:35:20.617 INFO:teuthology.orchestra.run.smithi187.stdout: python3-requests-2.25.1-8.el9.noarch 2024-03-29T04:35:20.617 INFO:teuthology.orchestra.run.smithi187.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-03-29T04:35:20.618 INFO:teuthology.orchestra.run.smithi187.stdout: python3-toml-0.10.2-6.el9.noarch 2024-03-29T04:35:20.618 INFO:teuthology.orchestra.run.smithi187.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-03-29T04:35:20.618 INFO:teuthology.orchestra.run.smithi187.stdout: python3-webob-1.8.7-6.el9.noarch 2024-03-29T04:35:20.618 INFO:teuthology.orchestra.run.smithi187.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-03-29T04:35:20.618 INFO:teuthology.orchestra.run.smithi187.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-03-29T04:35:20.618 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:20.618 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:35:20.988 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install ceph-base 2024-03-29T04:35:21.588 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:00:53 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:35:21.686 INFO:teuthology.orchestra.run.smithi187.stdout:Package ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:35:21.738 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:35:21.739 INFO:teuthology.orchestra.run.smithi187.stdout:Nothing to do. 2024-03-29T04:35:21.739 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:35:21.809 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install cephadm 2024-03-29T04:35:22.370 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:00:54 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:35:22.520 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:35:22.520 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:22.520 INFO:teuthology.orchestra.run.smithi187.stdout: Package Arch Version Repository Size 2024-03-29T04:35:22.521 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:22.521 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:35:22.521 INFO:teuthology.orchestra.run.smithi187.stdout: cephadm noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 82 k 2024-03-29T04:35:22.521 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:22.521 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:35:22.521 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:22.521 INFO:teuthology.orchestra.run.smithi187.stdout:Install 1 Package 2024-03-29T04:35:22.521 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:22.521 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 82 k 2024-03-29T04:35:22.521 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 372 k 2024-03-29T04:35:22.521 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:35:22.839 INFO:teuthology.orchestra.run.smithi187.stdout:cephadm-17.2.7-767.g9b4b8ffd.el9.noarch.rpm 258 kB/s | 82 kB 00:00 2024-03-29T04:35:22.840 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:35:22.840 INFO:teuthology.orchestra.run.smithi187.stdout:Total 257 kB/s | 82 kB 00:00 2024-03-29T04:35:22.840 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:35:22.845 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:35:22.845 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:35:22.847 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:35:22.847 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:35:22.938 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:35:23.371 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:35:23.485 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:35:23.857 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:35:24.110 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:35:24.111 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:24.111 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:35:24.111 INFO:teuthology.orchestra.run.smithi187.stdout: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:35:24.111 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:24.111 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:35:24.318 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install ceph-immutable-object-cache 2024-03-29T04:35:24.888 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:00:56 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:35:25.044 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:35:25.045 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:25.045 INFO:teuthology.orchestra.run.smithi187.stdout: Package Arch Version Repo Size 2024-03-29T04:35:25.045 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:25.045 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:35:25.045 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 137 k 2024-03-29T04:35:25.045 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:25.045 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:35:25.045 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:25.046 INFO:teuthology.orchestra.run.smithi187.stdout:Install 1 Package 2024-03-29T04:35:25.046 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:25.046 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 137 k 2024-03-29T04:35:25.046 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 379 k 2024-03-29T04:35:25.046 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:35:25.379 INFO:teuthology.orchestra.run.smithi187.stdout:ceph-immutable-object-cache-17.2.7-767.g9b4b8ff 413 kB/s | 137 kB 00:00 2024-03-29T04:35:25.380 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:35:25.380 INFO:teuthology.orchestra.run.smithi187.stdout:Total 411 kB/s | 137 kB 00:00 2024-03-29T04:35:25.380 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:35:25.387 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:35:25.388 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:35:25.441 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:35:25.441 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:35:25.595 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:35:25.647 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:35:25.671 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:35:25.671 INFO:teuthology.orchestra.run.smithi187.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:35:25.671 INFO:teuthology.orchestra.run.smithi187.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-03-29T04:35:25.672 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:26.645 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:35:26.645 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:26.645 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:35:26.645 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:35:26.646 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:26.646 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:35:26.939 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install ceph-mgr 2024-03-29T04:35:27.518 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:00:59 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:35:27.615 INFO:teuthology.orchestra.run.smithi187.stdout:Package ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:35:27.666 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:35:27.667 INFO:teuthology.orchestra.run.smithi187.stdout:Nothing to do. 2024-03-29T04:35:27.667 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:35:27.742 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install ceph-mgr-dashboard 2024-03-29T04:35:28.306 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:00 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:35:28.458 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:35:28.459 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:28.459 INFO:teuthology.orchestra.run.smithi187.stdout: Package Arch Version Repository Size 2024-03-29T04:35:28.459 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:28.459 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:35:28.459 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-mgr-dashboard noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 1.7 M 2024-03-29T04:35:28.459 INFO:teuthology.orchestra.run.smithi187.stdout:Installing dependencies: 2024-03-29T04:35:28.460 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-grafana-dashboards noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 23 k 2024-03-29T04:35:28.460 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-prometheus-alerts noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 15 k 2024-03-29T04:35:28.460 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-03-29T04:35:28.460 INFO:teuthology.orchestra.run.smithi187.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-03-29T04:35:28.460 INFO:teuthology.orchestra.run.smithi187.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-03-29T04:35:28.460 INFO:teuthology.orchestra.run.smithi187.stdout:Installing weak dependencies: 2024-03-29T04:35:28.460 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-03-29T04:35:28.460 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:28.460 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:35:28.460 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:28.460 INFO:teuthology.orchestra.run.smithi187.stdout:Install 7 Packages 2024-03-29T04:35:28.460 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:28.461 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 2.0 M 2024-03-29T04:35:28.461 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 8.1 M 2024-03-29T04:35:28.461 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:35:28.850 INFO:teuthology.orchestra.run.smithi187.stdout:(1/7): ceph-grafana-dashboards-17.2.7-767.g9b4b 117 kB/s | 23 kB 00:00 2024-03-29T04:35:28.941 INFO:teuthology.orchestra.run.smithi187.stdout:(2/7): ceph-prometheus-alerts-17.2.7-767.g9b4b8 50 kB/s | 15 kB 00:00 2024-03-29T04:35:29.033 INFO:teuthology.orchestra.run.smithi187.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 49 kB/s | 9.0 kB 00:00 2024-03-29T04:35:29.125 INFO:teuthology.orchestra.run.smithi187.stdout:(4/7): ceph-mgr-dashboard-17.2.7-767.g9b4b8ffd. 3.6 MB/s | 1.7 MB 00:00 2024-03-29T04:35:29.209 INFO:teuthology.orchestra.run.smithi187.stdout:(5/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 175 kB/s | 31 kB 00:00 2024-03-29T04:35:29.268 INFO:teuthology.orchestra.run.smithi187.stdout:(6/7): python3-jwt-2.4.0-1.el9.noarch.rpm 125 kB/s | 41 kB 00:00 2024-03-29T04:35:29.609 INFO:teuthology.orchestra.run.smithi187.stdout:(7/7): python3-routes-2.5.1-5.el9.noarch.rpm 390 kB/s | 188 kB 00:00 2024-03-29T04:35:29.611 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:35:29.612 INFO:teuthology.orchestra.run.smithi187.stdout:Total 1.7 MB/s | 2.0 MB 00:01 2024-03-29T04:35:29.612 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:35:29.620 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:35:29.620 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:35:29.686 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:35:29.687 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:35:29.984 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:35:30.052 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-03-29T04:35:30.086 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-03-29T04:35:30.133 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-03-29T04:35:30.165 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-03-29T04:35:30.193 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.no 5/7 2024-03-29T04:35:30.423 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.n 6/7 2024-03-29T04:35:30.455 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 7/7 2024-03-29T04:35:31.001 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 7/7 2024-03-29T04:35:31.001 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.n 1/7 2024-03-29T04:35:31.002 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 2/7 2024-03-29T04:35:31.002 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.no 3/7 2024-03-29T04:35:31.002 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-03-29T04:35:31.002 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-03-29T04:35:31.002 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-03-29T04:35:31.299 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-03-29T04:35:31.300 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:31.300 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:35:31.300 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:35:31.300 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:35:31.300 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:35:31.300 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-03-29T04:35:31.300 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-03-29T04:35:31.300 INFO:teuthology.orchestra.run.smithi187.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-03-29T04:35:31.300 INFO:teuthology.orchestra.run.smithi187.stdout: python3-routes-2.5.1-5.el9.noarch 2024-03-29T04:35:31.300 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:31.300 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:35:31.528 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install ceph-mgr-diskprediction-local 2024-03-29T04:35:32.091 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:04 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:35:32.242 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:35:32.243 INFO:teuthology.orchestra.run.smithi187.stdout:=========================================================================================== 2024-03-29T04:35:32.244 INFO:teuthology.orchestra.run.smithi187.stdout: Package Arch Version Repository Size 2024-03-29T04:35:32.244 INFO:teuthology.orchestra.run.smithi187.stdout:=========================================================================================== 2024-03-29T04:35:32.244 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:35:32.244 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 7.4 M 2024-03-29T04:35:32.244 INFO:teuthology.orchestra.run.smithi187.stdout:Installing dependencies: 2024-03-29T04:35:32.244 INFO:teuthology.orchestra.run.smithi187.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-03-29T04:35:32.244 INFO:teuthology.orchestra.run.smithi187.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-03-29T04:35:32.244 INFO:teuthology.orchestra.run.smithi187.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-03-29T04:35:32.244 INFO:teuthology.orchestra.run.smithi187.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-03-29T04:35:32.244 INFO:teuthology.orchestra.run.smithi187.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-03-29T04:35:32.244 INFO:teuthology.orchestra.run.smithi187.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-03-29T04:35:32.244 INFO:teuthology.orchestra.run.smithi187.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-03-29T04:35:32.244 INFO:teuthology.orchestra.run.smithi187.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-03-29T04:35:32.245 INFO:teuthology.orchestra.run.smithi187.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-03-29T04:35:32.245 INFO:teuthology.orchestra.run.smithi187.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-03-29T04:35:32.245 INFO:teuthology.orchestra.run.smithi187.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-03-29T04:35:32.245 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:32.245 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:35:32.245 INFO:teuthology.orchestra.run.smithi187.stdout:=========================================================================================== 2024-03-29T04:35:32.245 INFO:teuthology.orchestra.run.smithi187.stdout:Install 12 Packages 2024-03-29T04:35:32.245 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:32.246 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 37 M 2024-03-29T04:35:32.246 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 209 M 2024-03-29T04:35:32.246 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:35:32.575 INFO:teuthology.orchestra.run.smithi187.stdout:(1/12): flexiblas-3.0.4-8.el9.x86_64.rpm 224 kB/s | 32 kB 00:00 2024-03-29T04:35:32.608 INFO:teuthology.orchestra.run.smithi187.stdout:(2/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 532 kB/s | 17 kB 00:00 2024-03-29T04:35:32.641 INFO:teuthology.orchestra.run.smithi187.stdout:(3/12): openblas-0.3.21-2.el9.x86_64.rpm 1.0 MB/s | 34 kB 00:00 2024-03-29T04:35:32.701 INFO:teuthology.orchestra.run.smithi187.stdout:(4/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 11 MB/s | 3.0 MB 00:00 2024-03-29T04:35:32.917 INFO:teuthology.orchestra.run.smithi187.stdout:(5/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 17 MB/s | 4.7 MB 00:00 2024-03-29T04:35:32.984 INFO:teuthology.orchestra.run.smithi187.stdout:(6/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 17 MB/s | 4.8 MB 00:00 2024-03-29T04:35:33.001 INFO:teuthology.orchestra.run.smithi187.stdout:(7/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.6 MB/s | 217 kB 00:00 2024-03-29T04:35:33.360 INFO:teuthology.orchestra.run.smithi187.stdout:(8/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 42 MB/s | 16 MB 00:00 2024-03-29T04:35:33.843 INFO:teuthology.orchestra.run.smithi187.stdout:(9/12): libgfortran-11.4.1-3.el9.x86_64.rpm 955 kB/s | 803 kB 00:00 2024-03-29T04:35:33.877 INFO:teuthology.orchestra.run.smithi187.stdout:(10/12): libquadmath-11.4.1-3.el9.x86_64.rpm 378 kB/s | 195 kB 00:00 2024-03-29T04:35:33.994 INFO:teuthology.orchestra.run.smithi187.stdout:(11/12): python3-devel-3.9.17-2.el9.x86_64.rpm 1.6 MB/s | 245 kB 00:00 2024-03-29T04:35:34.277 INFO:teuthology.orchestra.run.smithi187.stdout:(12/12): ceph-mgr-diskprediction-local-17.2.7-7 4.0 MB/s | 7.4 MB 00:01 2024-03-29T04:35:34.279 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:35:34.280 INFO:teuthology.orchestra.run.smithi187.stdout:Total 18 MB/s | 37 MB 00:02 2024-03-29T04:35:34.322 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:35:34.345 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:35:34.345 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:35:34.776 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:35:34.776 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:35:35.483 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:35:35.567 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 1/12 2024-03-29T04:35:35.606 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-03-29T04:35:35.700 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 3/12 2024-03-29T04:35:35.743 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-03-29T04:35:35.929 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-03-29T04:35:35.959 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-03-29T04:35:36.038 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-03-29T04:35:36.683 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-03-29T04:35:36.797 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-03-29T04:35:38.068 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-03-29T04:35:39.110 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-03-29T04:35:39.141 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 12/12 2024-03-29T04:35:41.736 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 12/12 2024-03-29T04:35:41.736 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 1/12 2024-03-29T04:35:41.736 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 2/12 2024-03-29T04:35:41.736 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 3/12 2024-03-29T04:35:41.736 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 4/12 2024-03-29T04:35:41.736 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 5/12 2024-03-29T04:35:41.736 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-03-29T04:35:41.736 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 7/12 2024-03-29T04:35:41.736 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 8/12 2024-03-29T04:35:41.737 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 9/12 2024-03-29T04:35:41.737 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 10/12 2024-03-29T04:35:41.737 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 11/12 2024-03-29T04:35:42.161 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 12/12 2024-03-29T04:35:42.162 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:42.162 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:35:42.162 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:35:42.162 INFO:teuthology.orchestra.run.smithi187.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-03-29T04:35:42.162 INFO:teuthology.orchestra.run.smithi187.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-03-29T04:35:42.162 INFO:teuthology.orchestra.run.smithi187.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-03-29T04:35:42.162 INFO:teuthology.orchestra.run.smithi187.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-03-29T04:35:42.162 INFO:teuthology.orchestra.run.smithi187.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-03-29T04:35:42.162 INFO:teuthology.orchestra.run.smithi187.stdout: openblas-0.3.21-2.el9.x86_64 2024-03-29T04:35:42.162 INFO:teuthology.orchestra.run.smithi187.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-03-29T04:35:42.162 INFO:teuthology.orchestra.run.smithi187.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-03-29T04:35:42.162 INFO:teuthology.orchestra.run.smithi187.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-03-29T04:35:42.163 INFO:teuthology.orchestra.run.smithi187.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-03-29T04:35:42.163 INFO:teuthology.orchestra.run.smithi187.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-03-29T04:35:42.163 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:42.163 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:35:42.551 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install ceph-mgr-rook 2024-03-29T04:35:43.138 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:15 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:35:43.301 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:35:43.302 INFO:teuthology.orchestra.run.smithi187.stdout:======================================================================================= 2024-03-29T04:35:43.302 INFO:teuthology.orchestra.run.smithi187.stdout: Package Arch Version Repository Size 2024-03-29T04:35:43.302 INFO:teuthology.orchestra.run.smithi187.stdout:======================================================================================= 2024-03-29T04:35:43.302 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:35:43.302 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-mgr-rook noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 47 k 2024-03-29T04:35:43.303 INFO:teuthology.orchestra.run.smithi187.stdout:Installing dependencies: 2024-03-29T04:35:43.303 INFO:teuthology.orchestra.run.smithi187.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-03-29T04:35:43.303 INFO:teuthology.orchestra.run.smithi187.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-03-29T04:35:43.303 INFO:teuthology.orchestra.run.smithi187.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-03-29T04:35:43.303 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-03-29T04:35:43.303 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-03-29T04:35:43.303 INFO:teuthology.orchestra.run.smithi187.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-03-29T04:35:43.303 INFO:teuthology.orchestra.run.smithi187.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-03-29T04:35:43.303 INFO:teuthology.orchestra.run.smithi187.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-03-29T04:35:43.303 INFO:teuthology.orchestra.run.smithi187.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-03-29T04:35:43.303 INFO:teuthology.orchestra.run.smithi187.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-03-29T04:35:43.303 INFO:teuthology.orchestra.run.smithi187.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-03-29T04:35:43.304 INFO:teuthology.orchestra.run.smithi187.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-03-29T04:35:43.304 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:43.304 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:35:43.304 INFO:teuthology.orchestra.run.smithi187.stdout:======================================================================================= 2024-03-29T04:35:43.304 INFO:teuthology.orchestra.run.smithi187.stdout:Install 13 Packages 2024-03-29T04:35:43.304 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:43.304 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 2.0 M 2024-03-29T04:35:43.304 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 26 M 2024-03-29T04:35:43.305 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:35:43.507 INFO:teuthology.orchestra.run.smithi187.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 160 kB/s | 24 kB 00:00 2024-03-29T04:35:43.533 INFO:teuthology.orchestra.run.smithi187.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 102 kB/s | 18 kB 00:00 2024-03-29T04:35:43.591 INFO:teuthology.orchestra.run.smithi187.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.0 MB/s | 174 kB 00:00 2024-03-29T04:35:43.617 INFO:teuthology.orchestra.run.smithi187.stdout:(4/13): ceph-mgr-rook-17.2.7-767.g9b4b8ffd.el9. 181 kB/s | 47 kB 00:00 2024-03-29T04:35:43.650 INFO:teuthology.orchestra.run.smithi187.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.1 MB/s | 132 kB 00:00 2024-03-29T04:35:43.676 INFO:teuthology.orchestra.run.smithi187.stdout:(6/13): python3-pyasn1-modules-0.4.8-6.el9.noar 2.5 MB/s | 211 kB 00:00 2024-03-29T04:35:43.692 INFO:teuthology.orchestra.run.smithi187.stdout:(7/13): python3-requests-oauthlib-1.3.0-12.el9. 1.0 MB/s | 43 kB 00:00 2024-03-29T04:35:43.868 INFO:teuthology.orchestra.run.smithi187.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 148 kB/s | 32 kB 00:00 2024-03-29T04:35:43.885 INFO:teuthology.orchestra.run.smithi187.stdout:(9/13): python3-certifi-2021.10.8-2.el9.noarch. 72 kB/s | 15 kB 00:00 2024-03-29T04:35:43.926 INFO:teuthology.orchestra.run.smithi187.stdout:(10/13): python3-google-auth-2.28.2-1.el9.noarc 867 kB/s | 203 kB 00:00 2024-03-29T04:35:43.977 INFO:teuthology.orchestra.run.smithi187.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 643 kB/s | 59 kB 00:00 2024-03-29T04:35:44.069 INFO:teuthology.orchestra.run.smithi187.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 632 kB/s | 90 kB 00:00 2024-03-29T04:35:44.135 INFO:teuthology.orchestra.run.smithi187.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 3.8 MB/s | 1.0 MB 00:00 2024-03-29T04:35:44.140 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:35:44.140 INFO:teuthology.orchestra.run.smithi187.stdout:Total 2.4 MB/s | 2.0 MB 00:00 2024-03-29T04:35:44.140 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:35:44.153 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:35:44.153 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:35:44.242 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:35:44.242 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:35:44.756 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:35:44.915 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-03-29T04:35:44.978 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-03-29T04:35:45.042 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-03-29T04:35:45.080 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-03-29T04:35:45.121 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-03-29T04:35:45.214 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-03-29T04:35:45.314 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-03-29T04:35:45.364 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-03-29T04:35:46.092 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-03-29T04:35:46.136 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-03-29T04:35:46.170 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-03-29T04:35:46.217 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-03-29T04:35:46.239 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 13/13 2024-03-29T04:35:47.080 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 13/13 2024-03-29T04:35:47.080 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/13 2024-03-29T04:35:47.080 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-03-29T04:35:47.080 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-03-29T04:35:47.081 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-03-29T04:35:47.081 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-03-29T04:35:47.081 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-03-29T04:35:47.081 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-03-29T04:35:47.081 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-03-29T04:35:47.081 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-03-29T04:35:47.081 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-03-29T04:35:47.081 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-03-29T04:35:47.081 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-03-29T04:35:47.442 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-03-29T04:35:47.442 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:47.442 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:35:47.442 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:35:47.442 INFO:teuthology.orchestra.run.smithi187.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-03-29T04:35:47.443 INFO:teuthology.orchestra.run.smithi187.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-03-29T04:35:47.443 INFO:teuthology.orchestra.run.smithi187.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-03-29T04:35:47.443 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-03-29T04:35:47.443 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-03-29T04:35:47.443 INFO:teuthology.orchestra.run.smithi187.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-03-29T04:35:47.443 INFO:teuthology.orchestra.run.smithi187.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-03-29T04:35:47.443 INFO:teuthology.orchestra.run.smithi187.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-03-29T04:35:47.443 INFO:teuthology.orchestra.run.smithi187.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-03-29T04:35:47.443 INFO:teuthology.orchestra.run.smithi187.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-03-29T04:35:47.443 INFO:teuthology.orchestra.run.smithi187.stdout: python3-rsa-4.9-2.el9.noarch 2024-03-29T04:35:47.443 INFO:teuthology.orchestra.run.smithi187.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-03-29T04:35:47.443 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:47.443 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:35:47.711 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install ceph-mgr-cephadm 2024-03-29T04:35:48.276 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:20 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:35:48.426 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:35:48.427 INFO:teuthology.orchestra.run.smithi187.stdout:======================================================================================= 2024-03-29T04:35:48.427 INFO:teuthology.orchestra.run.smithi187.stdout: Package Arch Version Repository Size 2024-03-29T04:35:48.427 INFO:teuthology.orchestra.run.smithi187.stdout:======================================================================================= 2024-03-29T04:35:48.427 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:35:48.427 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-mgr-cephadm noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 123 k 2024-03-29T04:35:48.427 INFO:teuthology.orchestra.run.smithi187.stdout:Installing dependencies: 2024-03-29T04:35:48.427 INFO:teuthology.orchestra.run.smithi187.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-03-29T04:35:48.427 INFO:teuthology.orchestra.run.smithi187.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-03-29T04:35:48.427 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-03-29T04:35:48.427 INFO:teuthology.orchestra.run.smithi187.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-03-29T04:35:48.427 INFO:teuthology.orchestra.run.smithi187.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-03-29T04:35:48.427 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:48.428 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:35:48.428 INFO:teuthology.orchestra.run.smithi187.stdout:======================================================================================= 2024-03-29T04:35:48.428 INFO:teuthology.orchestra.run.smithi187.stdout:Install 6 Packages 2024-03-29T04:35:48.428 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:48.428 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 6.8 M 2024-03-29T04:35:48.428 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 33 M 2024-03-29T04:35:48.428 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:35:49.021 INFO:teuthology.orchestra.run.smithi187.stdout:(1/6): python3-babel-2.9.1-2.el9.noarch.rpm 21 MB/s | 5.8 MB 00:00 2024-03-29T04:35:49.046 INFO:teuthology.orchestra.run.smithi187.stdout:(2/6): ceph-mgr-cephadm-17.2.7-767.g9b4b8ffd.el 409 kB/s | 123 kB 00:00 2024-03-29T04:35:49.105 INFO:teuthology.orchestra.run.smithi187.stdout:(3/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 689 kB/s | 248 kB 00:00 2024-03-29T04:35:49.205 INFO:teuthology.orchestra.run.smithi187.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 364 kB/s | 58 kB 00:00 2024-03-29T04:35:49.264 INFO:teuthology.orchestra.run.smithi187.stdout:(5/6): python3-typing-extensions-4.4.0-2.el9.no 321 kB/s | 51 kB 00:00 2024-03-29T04:35:49.330 INFO:teuthology.orchestra.run.smithi187.stdout:(6/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 1.7 MB/s | 548 kB 00:00 2024-03-29T04:35:49.333 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:35:49.333 INFO:teuthology.orchestra.run.smithi187.stdout:Total 7.5 MB/s | 6.8 MB 00:00 2024-03-29T04:35:49.346 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:35:49.357 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:35:49.357 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:35:49.445 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:35:49.446 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:35:49.744 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:35:49.903 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-03-29T04:35:49.956 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-03-29T04:35:50.470 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-03-29T04:35:50.559 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-03-29T04:35:50.614 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-03-29T04:35:50.644 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 6/6 2024-03-29T04:35:51.466 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 6/6 2024-03-29T04:35:51.467 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/6 2024-03-29T04:35:51.467 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-03-29T04:35:51.467 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-03-29T04:35:51.467 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-03-29T04:35:51.467 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-03-29T04:35:51.755 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-03-29T04:35:51.755 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:51.755 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:35:51.755 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:35:51.755 INFO:teuthology.orchestra.run.smithi187.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-03-29T04:35:51.755 INFO:teuthology.orchestra.run.smithi187.stdout: python3-babel-2.9.1-2.el9.noarch 2024-03-29T04:35:51.755 INFO:teuthology.orchestra.run.smithi187.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-03-29T04:35:51.755 INFO:teuthology.orchestra.run.smithi187.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-03-29T04:35:51.755 INFO:teuthology.orchestra.run.smithi187.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-03-29T04:35:51.755 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:51.756 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:35:52.010 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install ceph-fuse 2024-03-29T04:35:52.567 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:24 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:35:52.718 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:35:52.718 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:52.719 INFO:teuthology.orchestra.run.smithi187.stdout: Package Arch Version Repository Size 2024-03-29T04:35:52.719 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:52.719 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:35:52.719 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-fuse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 812 k 2024-03-29T04:35:52.719 INFO:teuthology.orchestra.run.smithi187.stdout:Installing dependencies: 2024-03-29T04:35:52.719 INFO:teuthology.orchestra.run.smithi187.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-03-29T04:35:52.719 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:52.719 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:35:52.719 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:52.719 INFO:teuthology.orchestra.run.smithi187.stdout:Install 2 Packages 2024-03-29T04:35:52.719 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:52.720 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 891 k 2024-03-29T04:35:52.720 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 2.6 M 2024-03-29T04:35:52.720 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:35:52.840 INFO:teuthology.orchestra.run.smithi187.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 656 kB/s | 79 kB 00:00 2024-03-29T04:35:53.032 INFO:teuthology.orchestra.run.smithi187.stdout:(2/2): ceph-fuse-17.2.7-767.g9b4b8ffd.el9.x86_6 2.5 MB/s | 812 kB 00:00 2024-03-29T04:35:53.033 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:35:53.034 INFO:teuthology.orchestra.run.smithi187.stdout:Total 2.8 MB/s | 891 kB 00:00 2024-03-29T04:35:53.034 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:35:53.053 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:35:53.053 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:35:53.125 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:35:53.125 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:35:53.308 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:35:53.445 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-03-29T04:35:53.515 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/2 2024-03-29T04:35:54.215 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/2 2024-03-29T04:35:54.215 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/2 2024-03-29T04:35:54.514 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-03-29T04:35:54.514 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:54.514 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:35:54.514 INFO:teuthology.orchestra.run.smithi187.stdout: ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-03-29T04:35:54.515 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:54.515 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:35:54.786 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install librados-devel 2024-03-29T04:35:55.361 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:27 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:35:55.516 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:35:55.516 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:55.516 INFO:teuthology.orchestra.run.smithi187.stdout: Package Arch Version Repo Size 2024-03-29T04:35:55.517 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:55.517 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:35:55.517 INFO:teuthology.orchestra.run.smithi187.stdout: librados-devel x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 126 k 2024-03-29T04:35:55.517 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:55.517 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:35:55.517 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:55.517 INFO:teuthology.orchestra.run.smithi187.stdout:Install 1 Package 2024-03-29T04:35:55.517 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:55.517 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 126 k 2024-03-29T04:35:55.517 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 448 k 2024-03-29T04:35:55.517 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:35:55.736 INFO:teuthology.orchestra.run.smithi187.stdout:librados-devel-17.2.7-767.g9b4b8ffd.el9.x86_64. 578 kB/s | 126 kB 00:00 2024-03-29T04:35:55.737 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:35:55.737 INFO:teuthology.orchestra.run.smithi187.stdout:Total 574 kB/s | 126 kB 00:00 2024-03-29T04:35:55.737 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:35:55.742 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:35:55.742 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:35:55.798 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:35:55.798 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:35:55.940 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:35:56.034 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:35:56.490 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:35:56.806 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:35:56.806 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:56.807 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:35:56.807 INFO:teuthology.orchestra.run.smithi187.stdout: librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:35:56.807 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:56.807 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:35:57.048 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install libcephfs2 2024-03-29T04:35:57.619 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:29 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:35:57.719 INFO:teuthology.orchestra.run.smithi187.stdout:Package libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:35:57.772 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:35:57.773 INFO:teuthology.orchestra.run.smithi187.stdout:Nothing to do. 2024-03-29T04:35:57.773 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:35:57.846 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install libcephfs-devel 2024-03-29T04:35:58.413 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:30 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:35:58.563 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:35:58.564 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:58.564 INFO:teuthology.orchestra.run.smithi187.stdout: Package Arch Version Repo Size 2024-03-29T04:35:58.564 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:58.564 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:35:58.564 INFO:teuthology.orchestra.run.smithi187.stdout: libcephfs-devel x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 25 k 2024-03-29T04:35:58.564 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:58.564 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:35:58.564 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:35:58.564 INFO:teuthology.orchestra.run.smithi187.stdout:Install 1 Package 2024-03-29T04:35:58.564 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:58.564 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 25 k 2024-03-29T04:35:58.564 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 110 k 2024-03-29T04:35:58.565 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:35:58.823 INFO:teuthology.orchestra.run.smithi187.stdout:libcephfs-devel-17.2.7-767.g9b4b8ffd.el9.x86_64 97 kB/s | 25 kB 00:00 2024-03-29T04:35:58.823 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:35:58.824 INFO:teuthology.orchestra.run.smithi187.stdout:Total 96 kB/s | 25 kB 00:00 2024-03-29T04:35:58.824 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:35:58.827 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:35:58.827 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:35:58.828 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:35:58.829 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:35:58.920 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:35:59.036 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:35:59.406 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:35:59.670 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:35:59.671 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:59.671 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:35:59.671 INFO:teuthology.orchestra.run.smithi187.stdout: libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:35:59.671 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:35:59.671 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:35:59.855 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install librados2 2024-03-29T04:36:00.415 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:32 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:36:00.513 INFO:teuthology.orchestra.run.smithi187.stdout:Package librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:36:00.565 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:36:00.566 INFO:teuthology.orchestra.run.smithi187.stdout:Nothing to do. 2024-03-29T04:36:00.566 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:36:00.633 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install librbd1 2024-03-29T04:36:01.194 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:33 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:36:01.291 INFO:teuthology.orchestra.run.smithi187.stdout:Package librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:36:01.344 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:36:01.345 INFO:teuthology.orchestra.run.smithi187.stdout:Nothing to do. 2024-03-29T04:36:01.345 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:36:01.416 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install python3-rados 2024-03-29T04:36:01.985 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:33 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:36:02.084 INFO:teuthology.orchestra.run.smithi187.stdout:Package python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:36:02.136 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:36:02.138 INFO:teuthology.orchestra.run.smithi187.stdout:Nothing to do. 2024-03-29T04:36:02.138 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:36:02.208 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install python3-rgw 2024-03-29T04:36:02.772 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:34 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:36:02.871 INFO:teuthology.orchestra.run.smithi187.stdout:Package python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:36:02.922 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:36:02.923 INFO:teuthology.orchestra.run.smithi187.stdout:Nothing to do. 2024-03-29T04:36:02.924 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:36:02.992 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install python3-cephfs 2024-03-29T04:36:03.563 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:35 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:36:03.662 INFO:teuthology.orchestra.run.smithi187.stdout:Package python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:36:03.716 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:36:03.717 INFO:teuthology.orchestra.run.smithi187.stdout:Nothing to do. 2024-03-29T04:36:03.717 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:36:03.783 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install python3-rbd 2024-03-29T04:36:04.351 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:36 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:36:04.449 INFO:teuthology.orchestra.run.smithi187.stdout:Package python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:36:04.502 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:36:04.503 INFO:teuthology.orchestra.run.smithi187.stdout:Nothing to do. 2024-03-29T04:36:04.503 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:36:04.575 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install rbd-fuse 2024-03-29T04:36:05.140 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:37 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:36:05.291 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:36:05.291 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:36:05.291 INFO:teuthology.orchestra.run.smithi187.stdout: Package Architecture Version Repository Size 2024-03-29T04:36:05.291 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:36:05.292 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:36:05.292 INFO:teuthology.orchestra.run.smithi187.stdout: rbd-fuse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 82 k 2024-03-29T04:36:05.292 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:36:05.292 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:36:05.292 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:36:05.292 INFO:teuthology.orchestra.run.smithi187.stdout:Install 1 Package 2024-03-29T04:36:05.292 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:36:05.293 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 82 k 2024-03-29T04:36:05.293 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 222 k 2024-03-29T04:36:05.293 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:36:05.609 INFO:teuthology.orchestra.run.smithi187.stdout:rbd-fuse-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 260 kB/s | 82 kB 00:00 2024-03-29T04:36:05.610 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:36:05.610 INFO:teuthology.orchestra.run.smithi187.stdout:Total 259 kB/s | 82 kB 00:00 2024-03-29T04:36:05.610 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:36:05.617 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:36:05.617 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:36:05.672 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:36:05.673 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:36:05.809 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:36:05.915 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:36:06.329 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:36:06.637 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:36:06.637 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:36:06.637 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:36:06.637 INFO:teuthology.orchestra.run.smithi187.stdout: rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:36:06.637 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:36:06.638 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:36:06.877 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install rbd-mirror 2024-03-29T04:36:07.450 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:39 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:36:07.607 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:36:07.607 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:36:07.608 INFO:teuthology.orchestra.run.smithi187.stdout: Package Arch Version Repo Size 2024-03-29T04:36:07.608 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:36:07.608 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:36:07.608 INFO:teuthology.orchestra.run.smithi187.stdout: rbd-mirror x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.9 M 2024-03-29T04:36:07.608 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:36:07.608 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:36:07.608 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:36:07.608 INFO:teuthology.orchestra.run.smithi187.stdout:Install 1 Package 2024-03-29T04:36:07.608 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:36:07.608 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 2.9 M 2024-03-29T04:36:07.608 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 12 M 2024-03-29T04:36:07.608 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:36:07.995 INFO:teuthology.orchestra.run.smithi187.stdout:rbd-mirror-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 7.6 MB/s | 2.9 MB 00:00 2024-03-29T04:36:07.995 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:36:07.996 INFO:teuthology.orchestra.run.smithi187.stdout:Total 7.6 MB/s | 2.9 MB 00:00 2024-03-29T04:36:07.996 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:36:08.005 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:36:08.005 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:36:08.077 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:36:08.078 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:36:08.483 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:36:08.537 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:36:08.562 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:36:08.562 INFO:teuthology.orchestra.run.smithi187.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:36:08.562 INFO:teuthology.orchestra.run.smithi187.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-03-29T04:36:08.562 INFO:teuthology.orchestra.run.smithi187.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-29T04:36:08.562 INFO:teuthology.orchestra.run.smithi187.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-29T04:36:08.562 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:36:09.670 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:36:09.670 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:36:09.670 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:36:09.670 INFO:teuthology.orchestra.run.smithi187.stdout: rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:36:09.670 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:36:09.670 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:36:09.923 DEBUG:teuthology.orchestra.run.smithi187:> sudo yum -y install rbd-nbd 2024-03-29T04:36:10.502 INFO:teuthology.orchestra.run.smithi187.stdout:Last metadata expiration check: 0:01:42 ago on Fri 29 Mar 2024 04:34:28 AM UTC. 2024-03-29T04:36:10.653 INFO:teuthology.orchestra.run.smithi187.stdout:Dependencies resolved. 2024-03-29T04:36:10.654 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:36:10.654 INFO:teuthology.orchestra.run.smithi187.stdout: Package Architecture Version Repository Size 2024-03-29T04:36:10.655 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:36:10.655 INFO:teuthology.orchestra.run.smithi187.stdout:Installing: 2024-03-29T04:36:10.655 INFO:teuthology.orchestra.run.smithi187.stdout: rbd-nbd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 165 k 2024-03-29T04:36:10.655 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:36:10.655 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction Summary 2024-03-29T04:36:10.655 INFO:teuthology.orchestra.run.smithi187.stdout:================================================================================ 2024-03-29T04:36:10.655 INFO:teuthology.orchestra.run.smithi187.stdout:Install 1 Package 2024-03-29T04:36:10.655 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:36:10.655 INFO:teuthology.orchestra.run.smithi187.stdout:Total download size: 165 k 2024-03-29T04:36:10.655 INFO:teuthology.orchestra.run.smithi187.stdout:Installed size: 477 k 2024-03-29T04:36:10.655 INFO:teuthology.orchestra.run.smithi187.stdout:Downloading Packages: 2024-03-29T04:36:10.891 INFO:teuthology.orchestra.run.smithi187.stdout:rbd-nbd-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 700 kB/s | 165 kB 00:00 2024-03-29T04:36:10.892 INFO:teuthology.orchestra.run.smithi187.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:36:10.892 INFO:teuthology.orchestra.run.smithi187.stdout:Total 696 kB/s | 165 kB 00:00 2024-03-29T04:36:10.892 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction check 2024-03-29T04:36:10.900 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction check succeeded. 2024-03-29T04:36:10.900 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction test 2024-03-29T04:36:10.957 INFO:teuthology.orchestra.run.smithi187.stdout:Transaction test succeeded. 2024-03-29T04:36:10.957 INFO:teuthology.orchestra.run.smithi187.stdout:Running transaction 2024-03-29T04:36:11.101 INFO:teuthology.orchestra.run.smithi187.stdout: Preparing : 1/1 2024-03-29T04:36:11.199 INFO:teuthology.orchestra.run.smithi187.stdout: Installing : rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:36:11.632 INFO:teuthology.orchestra.run.smithi187.stdout: Running scriptlet: rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:36:11.928 INFO:teuthology.orchestra.run.smithi187.stdout: Verifying : rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:36:11.928 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:36:11.928 INFO:teuthology.orchestra.run.smithi187.stdout:Installed: 2024-03-29T04:36:11.928 INFO:teuthology.orchestra.run.smithi187.stdout: rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:36:11.928 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:36:11.928 INFO:teuthology.orchestra.run.smithi187.stdout:Complete! 2024-03-29T04:36:12.193 DEBUG:teuthology.parallel:result is None 2024-03-29T04:36:45.209 INFO:teuthology.orchestra.run.smithi181.stdout:[MIRROR] gperftools-libs-2.9.1-2.el9.x86_64.rpm: Curl error (28): Timeout was reached for http://epel.stl.us.ssimn.org/9/Everything/x86_64/Packages/g/gperftools-libs-2.9.1-2.el9.x86_64.rpm [Failed to connect to epel.stl.us.ssimn.org port 80: Connection timed out] 2024-03-29T04:37:23.394 INFO:teuthology.orchestra.run.smithi181.stdout:(24/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 1.8 kB/s | 309 kB 02:47 2024-03-29T04:37:23.395 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:37:23.395 INFO:teuthology.orchestra.run.smithi181.stdout:Total 347 kB/s | 58 MB 02:50 2024-03-29T04:37:23.396 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:37:23.430 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:37:23.430 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:37:23.893 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:37:23.893 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:37:25.125 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:37:25.472 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-03-29T04:37:25.538 INFO:teuthology.orchestra.run.smithi181.stdout: Upgrading : librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:37:25.619 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:37:25.707 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/26 2024-03-29T04:37:25.748 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:37:25.818 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:37:25.873 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-03-29T04:37:25.911 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-03-29T04:37:26.343 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-03-29T04:37:26.367 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 8/26 2024-03-29T04:37:26.413 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 8/26 2024-03-29T04:37:26.442 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-03-29T04:37:26.476 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x 10/26 2024-03-29T04:37:26.505 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 11/26 2024-03-29T04:37:26.569 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 12/26 2024-03-29T04:37:26.612 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:37:26.677 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:37:26.718 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-03-29T04:37:26.762 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-03-29T04:37:26.794 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-03-29T04:37:27.087 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-03-29T04:37:27.107 INFO:teuthology.orchestra.run.smithi181.stdout: Upgrading : librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 18/26 2024-03-29T04:37:27.182 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 18/26 2024-03-29T04:37:27.245 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 19/26 2024-03-29T04:37:27.268 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86 20/26 2024-03-29T04:37:29.645 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:37:29.690 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:37:30.202 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:37:30.254 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 22/26 2024-03-29T04:37:30.510 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 22/26 2024-03-29T04:37:30.510 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-03-29T04:37:30.511 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-03-29T04:37:30.511 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:37:30.595 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:37:39.669 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:37:39.669 INFO:teuthology.orchestra.run.smithi181.stdout:skipping the directory /sys 2024-03-29T04:37:39.669 INFO:teuthology.orchestra.run.smithi181.stdout:skipping the directory /proc 2024-03-29T04:37:39.669 INFO:teuthology.orchestra.run.smithi181.stdout:skipping the directory /mnt 2024-03-29T04:37:39.669 INFO:teuthology.orchestra.run.smithi181.stdout:skipping the directory /var/tmp 2024-03-29T04:37:39.669 INFO:teuthology.orchestra.run.smithi181.stdout:skipping the directory /home 2024-03-29T04:37:39.670 INFO:teuthology.orchestra.run.smithi181.stdout:skipping the directory /root 2024-03-29T04:37:39.670 INFO:teuthology.orchestra.run.smithi181.stdout:skipping the directory /tmp 2024-03-29T04:37:39.670 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:37:40.976 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 24/26 2024-03-29T04:37:41.053 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 24/26 2024-03-29T04:37:41.053 INFO:teuthology.orchestra.run.smithi181.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:37:41.053 INFO:teuthology.orchestra.run.smithi181.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-03-29T04:37:41.053 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-29T04:37:41.054 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-29T04:37:41.054 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:37:41.055 INFO:teuthology.orchestra.run.smithi181.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-03-29T04:37:41.128 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-03-29T04:37:41.128 INFO:teuthology.orchestra.run.smithi181.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:37:42.377 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:37:42.378 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/26 2024-03-29T04:37:42.378 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:37:42.378 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/26 2024-03-29T04:37:42.378 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:37:42.378 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/26 2024-03-29T04:37:42.378 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/26 2024-03-29T04:37:42.378 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 7/26 2024-03-29T04:37:42.378 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x 8/26 2024-03-29T04:37:42.378 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86 9/26 2024-03-29T04:37:42.378 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 10/26 2024-03-29T04:37:42.379 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 11/26 2024-03-29T04:37:42.379 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 12/26 2024-03-29T04:37:42.379 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:37:42.379 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-03-29T04:37:42.379 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-03-29T04:37:42.379 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-03-29T04:37:42.379 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-03-29T04:37:42.379 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-03-29T04:37:42.379 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-03-29T04:37:42.379 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-03-29T04:37:42.379 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-03-29T04:37:42.379 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-03-29T04:37:42.379 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:37:42.380 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-03-29T04:37:42.380 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 25/26 2024-03-29T04:37:43.384 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:37:43.385 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:37:43.385 INFO:teuthology.orchestra.run.smithi181.stdout:Upgraded: 2024-03-29T04:37:43.385 INFO:teuthology.orchestra.run.smithi181.stdout: librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:43.385 INFO:teuthology.orchestra.run.smithi181.stdout: librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:43.385 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:37:43.385 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:43.385 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:43.385 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:43.385 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:43.385 INFO:teuthology.orchestra.run.smithi181.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-03-29T04:37:43.385 INFO:teuthology.orchestra.run.smithi181.stdout: libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:43.385 INFO:teuthology.orchestra.run.smithi181.stdout: liboath-2.6.7-2.el9.x86_64 2024-03-29T04:37:43.385 INFO:teuthology.orchestra.run.smithi181.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-03-29T04:37:43.386 INFO:teuthology.orchestra.run.smithi181.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-03-29T04:37:43.386 INFO:teuthology.orchestra.run.smithi181.stdout: libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:43.386 INFO:teuthology.orchestra.run.smithi181.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-03-29T04:37:43.386 INFO:teuthology.orchestra.run.smithi181.stdout: librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:43.386 INFO:teuthology.orchestra.run.smithi181.stdout: libunwind-1.6.2-1.el9.x86_64 2024-03-29T04:37:43.386 INFO:teuthology.orchestra.run.smithi181.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-03-29T04:37:43.386 INFO:teuthology.orchestra.run.smithi181.stdout: python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:43.386 INFO:teuthology.orchestra.run.smithi181.stdout: python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:43.386 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:43.386 INFO:teuthology.orchestra.run.smithi181.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-03-29T04:37:43.386 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:43.386 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:43.386 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:43.386 INFO:teuthology.orchestra.run.smithi181.stdout: thrift-0.15.0-2.el9.x86_64 2024-03-29T04:37:43.387 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:37:43.387 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:37:43.729 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-test 2024-03-29T04:37:44.288 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:14 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:37:44.441 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:37:44.442 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:37:44.442 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2024-03-29T04:37:44.442 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:37:44.442 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:37:44.442 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-test x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 48 M 2024-03-29T04:37:44.442 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2024-03-29T04:37:44.442 INFO:teuthology.orchestra.run.smithi181.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-03-29T04:37:44.442 INFO:teuthology.orchestra.run.smithi181.stdout: libcephsqlite x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 163 k 2024-03-29T04:37:44.442 INFO:teuthology.orchestra.run.smithi181.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-AppStream 217 k 2024-03-29T04:37:44.443 INFO:teuthology.orchestra.run.smithi181.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-03-29T04:37:44.443 INFO:teuthology.orchestra.run.smithi181.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-03-29T04:37:44.443 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:37:44.443 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:37:44.443 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:37:44.443 INFO:teuthology.orchestra.run.smithi181.stdout:Install 6 Packages 2024-03-29T04:37:44.443 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:37:44.444 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 49 M 2024-03-29T04:37:44.444 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 193 M 2024-03-29T04:37:44.444 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:37:44.940 INFO:teuthology.orchestra.run.smithi181.stdout:(1/6): libcephsqlite-17.2.7-767.g9b4b8ffd.el9.x 617 kB/s | 163 kB 00:00 2024-03-29T04:37:44.974 INFO:teuthology.orchestra.run.smithi181.stdout:(2/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 729 kB/s | 217 kB 00:00 2024-03-29T04:37:45.007 INFO:teuthology.orchestra.run.smithi181.stdout:(3/6): socat-1.7.4.1-5.el9.x86_64.rpm 4.4 MB/s | 300 kB 00:00 2024-03-29T04:37:45.033 INFO:teuthology.orchestra.run.smithi181.stdout:(4/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.1 MB/s | 63 kB 00:00 2024-03-29T04:37:45.300 INFO:teuthology.orchestra.run.smithi181.stdout:(5/6): jq-1.6-16.el9.x86_64.rpm 641 kB/s | 187 kB 00:00 2024-03-29T04:37:47.433 INFO:teuthology.orchestra.run.smithi181.stdout:(6/6): ceph-test-17.2.7-767.g9b4b8ffd.el9.x86_6 17 MB/s | 48 MB 00:02 2024-03-29T04:37:47.436 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:37:47.436 INFO:teuthology.orchestra.run.smithi181.stdout:Total 16 MB/s | 49 MB 00:02 2024-03-29T04:37:47.451 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:37:47.471 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:37:47.471 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:37:47.868 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:37:47.868 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:37:48.439 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:37:48.523 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-03-29T04:37:48.564 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-03-29T04:37:48.600 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-03-29T04:37:48.641 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-03-29T04:37:48.669 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/6 2024-03-29T04:37:53.106 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/6 2024-03-29T04:37:53.185 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/6 2024-03-29T04:37:55.486 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/6 2024-03-29T04:37:55.486 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/6 2024-03-29T04:37:55.487 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/6 2024-03-29T04:37:55.487 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-03-29T04:37:55.487 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 4/6 2024-03-29T04:37:55.487 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/6 2024-03-29T04:37:55.839 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : jq-1.6-16.el9.x86_64 6/6 2024-03-29T04:37:55.840 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:37:55.840 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:37:55.840 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:55.840 INFO:teuthology.orchestra.run.smithi181.stdout: jq-1.6-16.el9.x86_64 2024-03-29T04:37:55.840 INFO:teuthology.orchestra.run.smithi181.stdout: libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:37:55.840 INFO:teuthology.orchestra.run.smithi181.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-03-29T04:37:55.840 INFO:teuthology.orchestra.run.smithi181.stdout: socat-1.7.4.1-5.el9.x86_64 2024-03-29T04:37:55.840 INFO:teuthology.orchestra.run.smithi181.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-03-29T04:37:55.840 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:37:55.840 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:37:56.136 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph 2024-03-29T04:37:56.702 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:26 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:37:56.857 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:37:56.859 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================== 2024-03-29T04:37:56.859 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2024-03-29T04:37:56.859 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================== 2024-03-29T04:37:56.859 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:37:56.860 INFO:teuthology.orchestra.run.smithi181.stdout: ceph x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 6.5 k 2024-03-29T04:37:56.860 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2024-03-29T04:37:56.860 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mds x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.1 M 2024-03-29T04:37:56.860 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 1.4 M 2024-03-29T04:37:56.860 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-modules-core noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 240 k 2024-03-29T04:37:56.860 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mon x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 3.9 M 2024-03-29T04:37:56.860 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-osd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 15 M 2024-03-29T04:37:56.860 INFO:teuthology.orchestra.run.smithi181.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-03-29T04:37:56.860 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-03-29T04:37:56.860 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-03-29T04:37:56.860 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-03-29T04:37:56.860 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-03-29T04:37:56.860 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-03-29T04:37:56.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-03-29T04:37:56.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-03-29T04:37:56.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-03-29T04:37:56.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-03-29T04:37:56.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-03-29T04:37:56.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-03-29T04:37:56.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-03-29T04:37:56.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-03-29T04:37:56.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-03-29T04:37:56.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-03-29T04:37:56.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-03-29T04:37:56.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-03-29T04:37:56.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-03-29T04:37:56.861 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-03-29T04:37:56.862 INFO:teuthology.orchestra.run.smithi181.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-03-29T04:37:56.862 INFO:teuthology.orchestra.run.smithi181.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-03-29T04:37:56.862 INFO:teuthology.orchestra.run.smithi181.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-03-29T04:37:56.862 INFO:teuthology.orchestra.run.smithi181.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-03-29T04:37:56.862 INFO:teuthology.orchestra.run.smithi181.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-03-29T04:37:56.862 INFO:teuthology.orchestra.run.smithi181.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-03-29T04:37:56.862 INFO:teuthology.orchestra.run.smithi181.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-03-29T04:37:56.862 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:37:56.862 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:37:56.862 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================== 2024-03-29T04:37:56.862 INFO:teuthology.orchestra.run.smithi181.stdout:Install 33 Packages 2024-03-29T04:37:56.862 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:37:56.863 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 27 M 2024-03-29T04:37:56.863 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 96 M 2024-03-29T04:37:56.863 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:37:57.202 INFO:teuthology.orchestra.run.smithi181.stdout:(1/33): ceph-17.2.7-767.g9b4b8ffd.el9.x86_64.rp 42 kB/s | 6.5 kB 00:00 2024-03-29T04:37:57.419 INFO:teuthology.orchestra.run.smithi181.stdout:(2/33): ceph-mgr-17.2.7-767.g9b4b8ffd.el9.x86_6 3.8 MB/s | 1.4 MB 00:00 2024-03-29T04:37:57.628 INFO:teuthology.orchestra.run.smithi181.stdout:(3/33): ceph-mon-17.2.7-767.g9b4b8ffd.el9.x86_6 9.2 MB/s | 3.9 MB 00:00 2024-03-29T04:37:57.703 INFO:teuthology.orchestra.run.smithi181.stdout:(4/33): ceph-mgr-modules-core-17.2.7-767.g9b4b8 3.1 MB/s | 240 kB 00:00 2024-03-29T04:37:57.813 INFO:teuthology.orchestra.run.smithi181.stdout:(5/33): python3-mako-1.1.4-6.el9.noarch.rpm 1.3 MB/s | 144 kB 00:00 2024-03-29T04:37:57.837 INFO:teuthology.orchestra.run.smithi181.stdout:(6/33): python3-markupsafe-1.1.1-12.el9.x86_64. 1.2 MB/s | 32 kB 00:00 2024-03-29T04:37:57.862 INFO:teuthology.orchestra.run.smithi181.stdout:(7/33): python3-pytz-2021.1-5.el9.noarch.rpm 1.8 MB/s | 47 kB 00:00 2024-03-29T04:37:57.887 INFO:teuthology.orchestra.run.smithi181.stdout:(8/33): python3-toml-0.10.2-6.el9.noarch.rpm 1.5 MB/s | 39 kB 00:00 2024-03-29T04:37:57.929 INFO:teuthology.orchestra.run.smithi181.stdout:(9/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 5.6 MB/s | 241 kB 00:00 2024-03-29T04:37:58.004 INFO:teuthology.orchestra.run.smithi181.stdout:(10/33): python3-cryptography-36.0.1-4.el9.x86_ 15 MB/s | 1.1 MB 00:00 2024-03-29T04:37:58.047 INFO:teuthology.orchestra.run.smithi181.stdout:(11/33): ceph-mds-17.2.7-767.g9b4b8ffd.el9.x86_ 2.1 MB/s | 2.1 MB 00:00 2024-03-29T04:37:58.072 INFO:teuthology.orchestra.run.smithi181.stdout:(12/33): python3-ply-3.11-14.el9.noarch.rpm 1.5 MB/s | 103 kB 00:00 2024-03-29T04:37:58.114 INFO:teuthology.orchestra.run.smithi181.stdout:(13/33): python3-urllib3-1.26.5-5.el9.noarch.rp 4.4 MB/s | 187 kB 00:00 2024-03-29T04:37:58.148 INFO:teuthology.orchestra.run.smithi181.stdout:(14/33): python3-pycparser-2.20-6.el9.noarch.rp 1.2 MB/s | 124 kB 00:00 2024-03-29T04:37:58.298 INFO:teuthology.orchestra.run.smithi181.stdout:(15/33): ceph-osd-17.2.7-767.g9b4b8ffd.el9.x86_ 17 MB/s | 15 MB 00:00 2024-03-29T04:37:58.407 INFO:teuthology.orchestra.run.smithi181.stdout:(16/33): python3-requests-2.25.1-8.el9.noarch.r 427 kB/s | 125 kB 00:00 2024-03-29T04:37:58.565 INFO:teuthology.orchestra.run.smithi181.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 104 kB/s | 43 kB 00:00 2024-03-29T04:37:58.632 INFO:teuthology.orchestra.run.smithi181.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 160 kB/s | 11 kB 00:00 2024-03-29T04:37:58.665 INFO:teuthology.orchestra.run.smithi181.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 468 kB/s | 172 kB 00:00 2024-03-29T04:37:58.691 INFO:teuthology.orchestra.run.smithi181.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 301 kB/s | 18 kB 00:00 2024-03-29T04:37:58.724 INFO:teuthology.orchestra.run.smithi181.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 395 kB/s | 23 kB 00:00 2024-03-29T04:37:58.750 INFO:teuthology.orchestra.run.smithi181.stdout:(22/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.0 MB/s | 358 kB 00:00 2024-03-29T04:37:58.775 INFO:teuthology.orchestra.run.smithi181.stdout:(23/33): python3-jaraco-functools-3.5.0-2.el9.n 232 kB/s | 19 kB 00:00 2024-03-29T04:37:58.800 INFO:teuthology.orchestra.run.smithi181.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 260 kB/s | 20 kB 00:00 2024-03-29T04:37:58.826 INFO:teuthology.orchestra.run.smithi181.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 612 kB/s | 46 kB 00:00 2024-03-29T04:37:58.859 INFO:teuthology.orchestra.run.smithi181.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 941 kB/s | 79 kB 00:00 2024-03-29T04:37:58.926 INFO:teuthology.orchestra.run.smithi181.stdout:(27/33): python3-pecan-1.4.2-3.el9.noarch.rpm 2.1 MB/s | 272 kB 00:00 2024-03-29T04:37:58.951 INFO:teuthology.orchestra.run.smithi181.stdout:(28/33): python3-portend-3.1.0-2.el9.noarch.rpm 131 kB/s | 16 kB 00:00 2024-03-29T04:37:58.968 INFO:teuthology.orchestra.run.smithi181.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 828 kB/s | 90 kB 00:00 2024-03-29T04:37:58.993 INFO:teuthology.orchestra.run.smithi181.stdout:(30/33): python3-tempora-5.0.0-2.el9.noarch.rpm 535 kB/s | 36 kB 00:00 2024-03-29T04:37:59.069 INFO:teuthology.orchestra.run.smithi181.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 2.2 MB/s | 230 kB 00:00 2024-03-29T04:37:59.102 INFO:teuthology.orchestra.run.smithi181.stdout:(32/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.1 MB/s | 427 kB 00:00 2024-03-29T04:37:59.127 INFO:teuthology.orchestra.run.smithi181.stdout:(33/33): python3-zc-lockfile-2.0-10.el9.noarch. 149 kB/s | 20 kB 00:00 2024-03-29T04:37:59.133 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:37:59.133 INFO:teuthology.orchestra.run.smithi181.stdout:Total 12 MB/s | 27 MB 00:02 2024-03-29T04:37:59.147 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:37:59.185 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:37:59.185 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:37:59.525 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:37:59.525 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:38:00.769 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:38:00.814 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-03-29T04:38:00.846 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-03-29T04:38:00.878 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-03-29T04:38:00.970 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-03-29T04:38:01.048 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-03-29T04:38:01.090 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-03-29T04:38:01.161 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-03-29T04:38:01.216 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 8/33 2024-03-29T04:38:01.254 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 9/33 2024-03-29T04:38:01.311 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-ply-3.11-14.el9.noarch 10/33 2024-03-29T04:38:01.373 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 11/33 2024-03-29T04:38:01.558 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 12/33 2024-03-29T04:38:01.616 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 13/33 2024-03-29T04:38:01.647 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 14/33 2024-03-29T04:38:01.681 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 15/33 2024-03-29T04:38:01.711 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 16/33 2024-03-29T04:38:01.795 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 17/33 2024-03-29T04:38:01.830 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-03-29T04:38:01.871 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 19/33 2024-03-29T04:38:01.912 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 20/33 2024-03-29T04:38:01.946 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 21/33 2024-03-29T04:38:01.981 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 22/33 2024-03-29T04:38:02.070 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 23/33 2024-03-29T04:38:02.103 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 24/33 2024-03-29T04:38:02.175 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 25/33 2024-03-29T04:38:02.311 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 26/33 2024-03-29T04:38:02.439 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-03-29T04:38:02.602 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.n 28/33 2024-03-29T04:38:02.634 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 29/33 2024-03-29T04:38:02.658 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 29/33 2024-03-29T04:38:02.658 INFO:teuthology.orchestra.run.smithi181.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:38:02.658 INFO:teuthology.orchestra.run.smithi181.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-03-29T04:38:02.659 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-29T04:38:02.659 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-29T04:38:02.659 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:04.035 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 30/33 2024-03-29T04:38:04.064 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 30/33 2024-03-29T04:38:04.064 INFO:teuthology.orchestra.run.smithi181.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:38:04.064 INFO:teuthology.orchestra.run.smithi181.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-03-29T04:38:04.065 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-29T04:38:04.065 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-29T04:38:04.065 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:04.461 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 31/33 2024-03-29T04:38:04.485 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 31/33 2024-03-29T04:38:04.485 INFO:teuthology.orchestra.run.smithi181.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:38:04.485 INFO:teuthology.orchestra.run.smithi181.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-03-29T04:38:04.485 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-29T04:38:04.485 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-29T04:38:04.486 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:04.686 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 32/33 2024-03-29T04:38:04.710 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 32/33 2024-03-29T04:38:04.710 INFO:teuthology.orchestra.run.smithi181.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:38:04.710 INFO:teuthology.orchestra.run.smithi181.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-03-29T04:38:04.710 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-29T04:38:04.710 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-29T04:38:04.710 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:04.776 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 33/33 2024-03-29T04:38:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 33/33 2024-03-29T04:38:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/33 2024-03-29T04:38:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/33 2024-03-29T04:38:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/33 2024-03-29T04:38:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/33 2024-03-29T04:38:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/33 2024-03-29T04:38:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.n 6/33 2024-03-29T04:38:06.235 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 7/33 2024-03-29T04:38:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 8/33 2024-03-29T04:38:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 9/33 2024-03-29T04:38:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 10/33 2024-03-29T04:38:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 11/33 2024-03-29T04:38:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 12/33 2024-03-29T04:38:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-ply-3.11-14.el9.noarch 13/33 2024-03-29T04:38:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 14/33 2024-03-29T04:38:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 15/33 2024-03-29T04:38:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 16/33 2024-03-29T04:38:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-03-29T04:38:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-03-29T04:38:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-03-29T04:38:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-03-29T04:38:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-03-29T04:38:06.236 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-03-29T04:38:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-03-29T04:38:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-03-29T04:38:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-03-29T04:38:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-03-29T04:38:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-03-29T04:38:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-03-29T04:38:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-03-29T04:38:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-03-29T04:38:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-03-29T04:38:06.237 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-03-29T04:38:07.095 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-03-29T04:38:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:07.096 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:38:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:38:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:38:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:38:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:38:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:38:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:38:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-03-29T04:38:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-03-29T04:38:07.096 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-03-29T04:38:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-03-29T04:38:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-03-29T04:38:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-03-29T04:38:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-03-29T04:38:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-03-29T04:38:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-03-29T04:38:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-03-29T04:38:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-03-29T04:38:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: python3-mako-1.1.4-6.el9.noarch 2024-03-29T04:38:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-03-29T04:38:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-03-29T04:38:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-03-29T04:38:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: python3-ply-3.11-14.el9.noarch 2024-03-29T04:38:07.097 INFO:teuthology.orchestra.run.smithi181.stdout: python3-portend-3.1.0-2.el9.noarch 2024-03-29T04:38:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-03-29T04:38:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pycparser-2.20-6.el9.noarch 2024-03-29T04:38:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pytz-2021.1-5.el9.noarch 2024-03-29T04:38:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: python3-requests-2.25.1-8.el9.noarch 2024-03-29T04:38:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-03-29T04:38:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: python3-toml-0.10.2-6.el9.noarch 2024-03-29T04:38:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-03-29T04:38:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: python3-webob-1.8.7-6.el9.noarch 2024-03-29T04:38:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-03-29T04:38:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-03-29T04:38:07.098 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:07.098 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:38:07.485 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-base 2024-03-29T04:38:08.058 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:38 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:38:08.157 INFO:teuthology.orchestra.run.smithi181.stdout:Package ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:38:08.209 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:38:08.210 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2024-03-29T04:38:08.210 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:38:08.290 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install cephadm 2024-03-29T04:38:08.849 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:38 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:38:08.999 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:38:09.000 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:38:09.000 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2024-03-29T04:38:09.001 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:38:09.001 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:38:09.001 INFO:teuthology.orchestra.run.smithi181.stdout: cephadm noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 82 k 2024-03-29T04:38:09.001 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:09.001 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:38:09.001 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:38:09.001 INFO:teuthology.orchestra.run.smithi181.stdout:Install 1 Package 2024-03-29T04:38:09.001 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:09.001 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 82 k 2024-03-29T04:38:09.001 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 372 k 2024-03-29T04:38:09.001 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:38:09.503 INFO:teuthology.orchestra.run.smithi181.stdout:cephadm-17.2.7-767.g9b4b8ffd.el9.noarch.rpm 164 kB/s | 82 kB 00:00 2024-03-29T04:38:09.504 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:38:09.504 INFO:teuthology.orchestra.run.smithi181.stdout:Total 163 kB/s | 82 kB 00:00 2024-03-29T04:38:09.504 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:38:09.509 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:38:09.509 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:38:09.511 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:38:09.511 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:38:09.586 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:38:09.983 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:38:10.081 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:38:10.454 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:38:10.719 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:38:10.719 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:10.720 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:38:10.720 INFO:teuthology.orchestra.run.smithi181.stdout: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:38:10.720 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:10.720 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:38:10.924 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-immutable-object-cache 2024-03-29T04:38:11.504 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:41 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:38:11.662 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:38:11.663 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:38:11.663 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repo Size 2024-03-29T04:38:11.664 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:38:11.664 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:38:11.664 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 137 k 2024-03-29T04:38:11.664 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:11.664 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:38:11.664 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:38:11.664 INFO:teuthology.orchestra.run.smithi181.stdout:Install 1 Package 2024-03-29T04:38:11.664 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:11.664 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 137 k 2024-03-29T04:38:11.664 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 379 k 2024-03-29T04:38:11.664 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:38:11.909 INFO:teuthology.orchestra.run.smithi181.stdout:ceph-immutable-object-cache-17.2.7-767.g9b4b8ff 560 kB/s | 137 kB 00:00 2024-03-29T04:38:11.910 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:38:11.910 INFO:teuthology.orchestra.run.smithi181.stdout:Total 557 kB/s | 137 kB 00:00 2024-03-29T04:38:11.910 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:38:11.918 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:38:11.918 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:38:11.971 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:38:11.971 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:38:12.122 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:38:12.177 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:38:12.202 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:38:12.202 INFO:teuthology.orchestra.run.smithi181.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:38:12.202 INFO:teuthology.orchestra.run.smithi181.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-03-29T04:38:12.202 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:13.157 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:38:13.157 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:13.157 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:38:13.157 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:38:13.157 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:13.157 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:38:13.404 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-mgr 2024-03-29T04:38:13.980 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:43 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:38:14.078 INFO:teuthology.orchestra.run.smithi181.stdout:Package ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:38:14.129 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:38:14.129 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2024-03-29T04:38:14.130 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:38:14.219 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-mgr-dashboard 2024-03-29T04:38:14.779 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:44 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:38:14.929 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:38:14.930 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:38:14.930 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2024-03-29T04:38:14.930 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:38:14.930 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:38:14.930 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-dashboard noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 1.7 M 2024-03-29T04:38:14.931 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2024-03-29T04:38:14.931 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-grafana-dashboards noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 23 k 2024-03-29T04:38:14.931 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-prometheus-alerts noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 15 k 2024-03-29T04:38:14.931 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-03-29T04:38:14.931 INFO:teuthology.orchestra.run.smithi181.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-03-29T04:38:14.931 INFO:teuthology.orchestra.run.smithi181.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-03-29T04:38:14.931 INFO:teuthology.orchestra.run.smithi181.stdout:Installing weak dependencies: 2024-03-29T04:38:14.931 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-03-29T04:38:14.931 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:14.931 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:38:14.931 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:38:14.931 INFO:teuthology.orchestra.run.smithi181.stdout:Install 7 Packages 2024-03-29T04:38:14.931 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:14.932 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 2.0 M 2024-03-29T04:38:14.932 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 8.1 M 2024-03-29T04:38:14.932 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:38:15.480 INFO:teuthology.orchestra.run.smithi181.stdout:(1/7): ceph-prometheus-alerts-17.2.7-767.g9b4b8 79 kB/s | 15 kB 00:00 2024-03-29T04:38:15.573 INFO:teuthology.orchestra.run.smithi181.stdout:(2/7): ceph-grafana-dashboards-17.2.7-767.g9b4b 85 kB/s | 23 kB 00:00 2024-03-29T04:38:15.680 INFO:teuthology.orchestra.run.smithi181.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 45 kB/s | 9.0 kB 00:00 2024-03-29T04:38:15.781 INFO:teuthology.orchestra.run.smithi181.stdout:(4/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 311 kB/s | 31 kB 00:00 2024-03-29T04:38:15.831 INFO:teuthology.orchestra.run.smithi181.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 158 kB/s | 41 kB 00:00 2024-03-29T04:38:15.923 INFO:teuthology.orchestra.run.smithi181.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 1.3 MB/s | 188 kB 00:00 2024-03-29T04:38:16.615 INFO:teuthology.orchestra.run.smithi181.stdout:(7/7): ceph-mgr-dashboard-17.2.7-767.g9b4b8ffd. 1.3 MB/s | 1.7 MB 00:01 2024-03-29T04:38:16.619 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:38:16.619 INFO:teuthology.orchestra.run.smithi181.stdout:Total 1.2 MB/s | 2.0 MB 00:01 2024-03-29T04:38:16.619 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:38:16.627 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:38:16.627 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:38:16.688 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:38:16.689 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:38:16.992 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:38:17.058 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-03-29T04:38:17.100 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-03-29T04:38:17.147 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-03-29T04:38:17.187 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-03-29T04:38:17.216 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.no 5/7 2024-03-29T04:38:17.444 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.n 6/7 2024-03-29T04:38:17.477 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 7/7 2024-03-29T04:38:17.969 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 7/7 2024-03-29T04:38:17.969 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.n 1/7 2024-03-29T04:38:17.969 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 2/7 2024-03-29T04:38:17.969 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.no 3/7 2024-03-29T04:38:17.969 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-03-29T04:38:17.969 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-03-29T04:38:17.969 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-03-29T04:38:18.255 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-03-29T04:38:18.255 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:18.256 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:38:18.256 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:38:18.256 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:38:18.256 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:38:18.256 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-03-29T04:38:18.256 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-03-29T04:38:18.256 INFO:teuthology.orchestra.run.smithi181.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-03-29T04:38:18.257 INFO:teuthology.orchestra.run.smithi181.stdout: python3-routes-2.5.1-5.el9.noarch 2024-03-29T04:38:18.257 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:18.257 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:38:18.495 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-mgr-diskprediction-local 2024-03-29T04:38:19.061 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:49 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:38:19.212 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:38:19.213 INFO:teuthology.orchestra.run.smithi181.stdout:=========================================================================================== 2024-03-29T04:38:19.214 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2024-03-29T04:38:19.214 INFO:teuthology.orchestra.run.smithi181.stdout:=========================================================================================== 2024-03-29T04:38:19.214 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:38:19.214 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 7.4 M 2024-03-29T04:38:19.214 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2024-03-29T04:38:19.214 INFO:teuthology.orchestra.run.smithi181.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-03-29T04:38:19.214 INFO:teuthology.orchestra.run.smithi181.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-03-29T04:38:19.214 INFO:teuthology.orchestra.run.smithi181.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-03-29T04:38:19.214 INFO:teuthology.orchestra.run.smithi181.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-03-29T04:38:19.214 INFO:teuthology.orchestra.run.smithi181.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-03-29T04:38:19.214 INFO:teuthology.orchestra.run.smithi181.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-03-29T04:38:19.214 INFO:teuthology.orchestra.run.smithi181.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-03-29T04:38:19.214 INFO:teuthology.orchestra.run.smithi181.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-03-29T04:38:19.215 INFO:teuthology.orchestra.run.smithi181.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-03-29T04:38:19.215 INFO:teuthology.orchestra.run.smithi181.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-03-29T04:38:19.215 INFO:teuthology.orchestra.run.smithi181.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-03-29T04:38:19.215 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:19.215 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:38:19.215 INFO:teuthology.orchestra.run.smithi181.stdout:=========================================================================================== 2024-03-29T04:38:19.215 INFO:teuthology.orchestra.run.smithi181.stdout:Install 12 Packages 2024-03-29T04:38:19.215 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:19.216 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 37 M 2024-03-29T04:38:19.216 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 209 M 2024-03-29T04:38:19.216 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:38:19.497 INFO:teuthology.orchestra.run.smithi181.stdout:(1/12): flexiblas-3.0.4-8.el9.x86_64.rpm 253 kB/s | 32 kB 00:00 2024-03-29T04:38:19.530 INFO:teuthology.orchestra.run.smithi181.stdout:(2/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 531 kB/s | 17 kB 00:00 2024-03-29T04:38:19.597 INFO:teuthology.orchestra.run.smithi181.stdout:(3/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 13 MB/s | 3.0 MB 00:00 2024-03-29T04:38:19.623 INFO:teuthology.orchestra.run.smithi181.stdout:(4/12): openblas-0.3.21-2.el9.x86_64.rpm 379 kB/s | 34 kB 00:00 2024-03-29T04:38:19.831 INFO:teuthology.orchestra.run.smithi181.stdout:(5/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 20 MB/s | 4.7 MB 00:00 2024-03-29T04:38:19.907 INFO:teuthology.orchestra.run.smithi181.stdout:(6/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 17 MB/s | 4.8 MB 00:00 2024-03-29T04:38:19.932 INFO:teuthology.orchestra.run.smithi181.stdout:(7/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.1 MB/s | 217 kB 00:00 2024-03-29T04:38:20.391 INFO:teuthology.orchestra.run.smithi181.stdout:(8/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 32 MB/s | 16 MB 00:00 2024-03-29T04:38:20.483 INFO:teuthology.orchestra.run.smithi181.stdout:(9/12): libgfortran-11.4.1-3.el9.x86_64.rpm 1.4 MB/s | 803 kB 00:00 2024-03-29T04:38:20.625 INFO:teuthology.orchestra.run.smithi181.stdout:(10/12): libquadmath-11.4.1-3.el9.x86_64.rpm 837 kB/s | 195 kB 00:00 2024-03-29T04:38:20.775 INFO:teuthology.orchestra.run.smithi181.stdout:(11/12): ceph-mgr-diskprediction-local-17.2.7-7 5.3 MB/s | 7.4 MB 00:01 2024-03-29T04:38:20.800 INFO:teuthology.orchestra.run.smithi181.stdout:(12/12): python3-devel-3.9.17-2.el9.x86_64.rpm 774 kB/s | 245 kB 00:00 2024-03-29T04:38:20.803 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:38:20.803 INFO:teuthology.orchestra.run.smithi181.stdout:Total 23 MB/s | 37 MB 00:01 2024-03-29T04:38:20.846 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:38:20.870 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:38:20.870 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:38:21.294 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:38:21.294 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:38:21.986 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:38:22.065 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 1/12 2024-03-29T04:38:22.096 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-03-29T04:38:22.188 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 3/12 2024-03-29T04:38:22.232 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-03-29T04:38:22.404 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-03-29T04:38:22.432 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-03-29T04:38:22.515 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-03-29T04:38:23.146 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-03-29T04:38:23.254 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-03-29T04:38:24.491 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-03-29T04:38:25.507 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-03-29T04:38:25.539 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 12/12 2024-03-29T04:38:28.073 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 12/12 2024-03-29T04:38:28.073 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 1/12 2024-03-29T04:38:28.073 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 2/12 2024-03-29T04:38:28.073 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 3/12 2024-03-29T04:38:28.073 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 4/12 2024-03-29T04:38:28.073 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 5/12 2024-03-29T04:38:28.073 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-03-29T04:38:28.073 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 7/12 2024-03-29T04:38:28.073 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 8/12 2024-03-29T04:38:28.073 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 9/12 2024-03-29T04:38:28.073 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 10/12 2024-03-29T04:38:28.073 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 11/12 2024-03-29T04:38:28.454 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 12/12 2024-03-29T04:38:28.454 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:28.454 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:38:28.454 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:38:28.454 INFO:teuthology.orchestra.run.smithi181.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-03-29T04:38:28.454 INFO:teuthology.orchestra.run.smithi181.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-03-29T04:38:28.455 INFO:teuthology.orchestra.run.smithi181.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-03-29T04:38:28.455 INFO:teuthology.orchestra.run.smithi181.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-03-29T04:38:28.455 INFO:teuthology.orchestra.run.smithi181.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-03-29T04:38:28.455 INFO:teuthology.orchestra.run.smithi181.stdout: openblas-0.3.21-2.el9.x86_64 2024-03-29T04:38:28.455 INFO:teuthology.orchestra.run.smithi181.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-03-29T04:38:28.455 INFO:teuthology.orchestra.run.smithi181.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-03-29T04:38:28.455 INFO:teuthology.orchestra.run.smithi181.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-03-29T04:38:28.455 INFO:teuthology.orchestra.run.smithi181.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-03-29T04:38:28.455 INFO:teuthology.orchestra.run.smithi181.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-03-29T04:38:28.455 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:28.455 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:38:28.824 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-mgr-rook 2024-03-29T04:38:29.399 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:03:59 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:38:29.551 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:38:29.552 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================= 2024-03-29T04:38:29.552 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2024-03-29T04:38:29.553 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================= 2024-03-29T04:38:29.553 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:38:29.553 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-rook noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 47 k 2024-03-29T04:38:29.553 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2024-03-29T04:38:29.553 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-03-29T04:38:29.553 INFO:teuthology.orchestra.run.smithi181.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-03-29T04:38:29.553 INFO:teuthology.orchestra.run.smithi181.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-03-29T04:38:29.553 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-03-29T04:38:29.553 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-03-29T04:38:29.553 INFO:teuthology.orchestra.run.smithi181.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-03-29T04:38:29.553 INFO:teuthology.orchestra.run.smithi181.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-03-29T04:38:29.553 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-03-29T04:38:29.553 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-03-29T04:38:29.553 INFO:teuthology.orchestra.run.smithi181.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-03-29T04:38:29.553 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-03-29T04:38:29.554 INFO:teuthology.orchestra.run.smithi181.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-03-29T04:38:29.554 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:29.554 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:38:29.554 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================= 2024-03-29T04:38:29.554 INFO:teuthology.orchestra.run.smithi181.stdout:Install 13 Packages 2024-03-29T04:38:29.554 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:29.554 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 2.0 M 2024-03-29T04:38:29.555 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 26 M 2024-03-29T04:38:29.555 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:38:30.033 INFO:teuthology.orchestra.run.smithi181.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 147 kB/s | 24 kB 00:00 2024-03-29T04:38:30.058 INFO:teuthology.orchestra.run.smithi181.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 95 kB/s | 18 kB 00:00 2024-03-29T04:38:30.175 INFO:teuthology.orchestra.run.smithi181.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.2 MB/s | 174 kB 00:00 2024-03-29T04:38:30.200 INFO:teuthology.orchestra.run.smithi181.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 931 kB/s | 132 kB 00:00 2024-03-29T04:38:30.250 INFO:teuthology.orchestra.run.smithi181.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 2.7 MB/s | 211 kB 00:00 2024-03-29T04:38:30.267 INFO:teuthology.orchestra.run.smithi181.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 639 kB/s | 43 kB 00:00 2024-03-29T04:38:30.284 INFO:teuthology.orchestra.run.smithi181.stdout:(7/13): ceph-mgr-rook-17.2.7-767.g9b4b8ffd.el9. 114 kB/s | 47 kB 00:00 2024-03-29T04:38:30.418 INFO:teuthology.orchestra.run.smithi181.stdout:(8/13): python3-google-auth-2.28.2-1.el9.noarch 1.5 MB/s | 203 kB 00:00 2024-03-29T04:38:30.435 INFO:teuthology.orchestra.run.smithi181.stdout:(9/13): python3-certifi-2021.10.8-2.el9.noarch. 90 kB/s | 15 kB 00:00 2024-03-29T04:38:30.451 INFO:teuthology.orchestra.run.smithi181.stdout:(10/13): python3-cachetools-4.2.4-1.el9.noarch. 160 kB/s | 32 kB 00:00 2024-03-29T04:38:30.494 INFO:teuthology.orchestra.run.smithi181.stdout:(11/13): python3-websocket-client-1.2.3-2.el9.n 2.1 MB/s | 90 kB 00:00 2024-03-29T04:38:30.577 INFO:teuthology.orchestra.run.smithi181.stdout:(12/13): python3-rsa-4.9-2.el9.noarch.rpm 416 kB/s | 59 kB 00:00 2024-03-29T04:38:30.961 INFO:teuthology.orchestra.run.smithi181.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 1.9 MB/s | 1.0 MB 00:00 2024-03-29T04:38:30.962 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:38:30.963 INFO:teuthology.orchestra.run.smithi181.stdout:Total 1.5 MB/s | 2.0 MB 00:01 2024-03-29T04:38:30.963 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:38:30.977 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:38:30.977 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:38:31.072 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:38:31.072 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:38:31.608 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:38:31.767 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-03-29T04:38:31.828 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-03-29T04:38:31.892 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-03-29T04:38:31.930 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-03-29T04:38:31.971 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-03-29T04:38:32.065 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-03-29T04:38:32.182 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-03-29T04:38:32.248 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-03-29T04:38:32.972 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-03-29T04:38:33.019 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-03-29T04:38:33.053 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-03-29T04:38:33.099 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-03-29T04:38:33.123 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 13/13 2024-03-29T04:38:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 13/13 2024-03-29T04:38:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/13 2024-03-29T04:38:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-03-29T04:38:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-03-29T04:38:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-03-29T04:38:33.905 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-03-29T04:38:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-03-29T04:38:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-03-29T04:38:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-03-29T04:38:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-03-29T04:38:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-03-29T04:38:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-03-29T04:38:33.906 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-03-29T04:38:34.262 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-03-29T04:38:34.262 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:34.262 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:38:34.262 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:38:34.262 INFO:teuthology.orchestra.run.smithi181.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-03-29T04:38:34.262 INFO:teuthology.orchestra.run.smithi181.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-03-29T04:38:34.263 INFO:teuthology.orchestra.run.smithi181.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-03-29T04:38:34.263 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-03-29T04:38:34.263 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-03-29T04:38:34.263 INFO:teuthology.orchestra.run.smithi181.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-03-29T04:38:34.263 INFO:teuthology.orchestra.run.smithi181.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-03-29T04:38:34.263 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-03-29T04:38:34.263 INFO:teuthology.orchestra.run.smithi181.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-03-29T04:38:34.263 INFO:teuthology.orchestra.run.smithi181.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-03-29T04:38:34.263 INFO:teuthology.orchestra.run.smithi181.stdout: python3-rsa-4.9-2.el9.noarch 2024-03-29T04:38:34.263 INFO:teuthology.orchestra.run.smithi181.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-03-29T04:38:34.263 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:34.264 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:38:34.501 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-mgr-cephadm 2024-03-29T04:38:35.100 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:04:05 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:38:35.258 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:38:35.259 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================= 2024-03-29T04:38:35.259 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2024-03-29T04:38:35.259 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================= 2024-03-29T04:38:35.259 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:38:35.259 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-cephadm noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 123 k 2024-03-29T04:38:35.259 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2024-03-29T04:38:35.259 INFO:teuthology.orchestra.run.smithi181.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-03-29T04:38:35.260 INFO:teuthology.orchestra.run.smithi181.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-03-29T04:38:35.260 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-03-29T04:38:35.260 INFO:teuthology.orchestra.run.smithi181.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-03-29T04:38:35.260 INFO:teuthology.orchestra.run.smithi181.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-03-29T04:38:35.260 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:35.260 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:38:35.260 INFO:teuthology.orchestra.run.smithi181.stdout:======================================================================================= 2024-03-29T04:38:35.260 INFO:teuthology.orchestra.run.smithi181.stdout:Install 6 Packages 2024-03-29T04:38:35.260 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:38:35.261 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 6.8 M 2024-03-29T04:38:35.261 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 33 M 2024-03-29T04:38:35.261 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:38:36.055 INFO:teuthology.orchestra.run.smithi181.stdout:(1/6): python3-babel-2.9.1-2.el9.noarch.rpm 20 MB/s | 5.8 MB 00:00 2024-03-29T04:38:36.080 INFO:teuthology.orchestra.run.smithi181.stdout:(2/6): ceph-mgr-cephadm-17.2.7-767.g9b4b8ffd.el 384 kB/s | 123 kB 00:00 2024-03-29T04:38:36.147 INFO:teuthology.orchestra.run.smithi181.stdout:(3/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 639 kB/s | 248 kB 00:00 2024-03-29T04:38:57.603 INFO:teuthology.orchestra.run.smithi181.stdout:(4/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 25 kB/s | 548 kB 00:21 2024-03-29T04:39:42.807 INFO:teuthology.orchestra.run.smithi181.stdout:(5/6): python3-natsort-7.1.1-5.el9.noarch.rpm 885 B/s | 58 kB 01:06 2024-03-29T04:40:47.353 INFO:teuthology.orchestra.run.smithi181.stdout:[MIRROR] python3-typing-extensions-4.4.0-2.el9.noarch.rpm: Curl error (28): Timeout was reached for http://epel.stl.us.ssimn.org/9/Everything/x86_64/Packages/p/python3-typing-extensions-4.4.0-2.el9.noarch.rpm [Failed to connect to epel.stl.us.ssimn.org port 80: Connection timed out] 2024-03-29T04:42:58.147 INFO:teuthology.orchestra.run.smithi181.stdout:[MIRROR] python3-typing-extensions-4.4.0-2.el9.noarch.rpm: Curl error (28): Timeout was reached for https://epel.stl.us.ssimn.org/9/Everything/x86_64/Packages/p/python3-typing-extensions-4.4.0-2.el9.noarch.rpm [Failed to connect to epel.stl.us.ssimn.org port 443: Connection timed out] 2024-03-29T04:42:58.374 INFO:teuthology.orchestra.run.smithi181.stdout:(6/6): python3-typing-extensions-4.4.0-2.el9.no 199 B/s | 51 kB 04:22 2024-03-29T04:42:58.374 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:42:58.375 INFO:teuthology.orchestra.run.smithi181.stdout:Total 26 kB/s | 6.8 MB 04:23 2024-03-29T04:42:58.388 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:42:58.400 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:42:58.400 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:42:58.491 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:42:58.492 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:42:58.785 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:42:58.947 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-03-29T04:42:59.000 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-03-29T04:42:59.514 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-03-29T04:42:59.602 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-03-29T04:42:59.658 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-03-29T04:42:59.688 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 6/6 2024-03-29T04:43:00.459 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 6/6 2024-03-29T04:43:00.460 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/6 2024-03-29T04:43:00.460 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-03-29T04:43:00.460 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-03-29T04:43:00.460 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-03-29T04:43:00.460 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-03-29T04:43:00.741 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-03-29T04:43:00.741 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:00.741 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:43:00.741 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:43:00.741 INFO:teuthology.orchestra.run.smithi181.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-03-29T04:43:00.741 INFO:teuthology.orchestra.run.smithi181.stdout: python3-babel-2.9.1-2.el9.noarch 2024-03-29T04:43:00.741 INFO:teuthology.orchestra.run.smithi181.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-03-29T04:43:00.742 INFO:teuthology.orchestra.run.smithi181.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-03-29T04:43:00.742 INFO:teuthology.orchestra.run.smithi181.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-03-29T04:43:00.742 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:00.742 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:43:00.996 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install ceph-fuse 2024-03-29T04:43:01.557 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:08:31 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:43:01.707 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:43:01.708 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:01.708 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repository Size 2024-03-29T04:43:01.708 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:01.708 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:43:01.708 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-fuse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 812 k 2024-03-29T04:43:01.708 INFO:teuthology.orchestra.run.smithi181.stdout:Installing dependencies: 2024-03-29T04:43:01.708 INFO:teuthology.orchestra.run.smithi181.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-03-29T04:43:01.708 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:01.708 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:43:01.708 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:01.709 INFO:teuthology.orchestra.run.smithi181.stdout:Install 2 Packages 2024-03-29T04:43:01.709 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:01.709 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 891 k 2024-03-29T04:43:01.709 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 2.6 M 2024-03-29T04:43:01.709 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:43:01.842 INFO:teuthology.orchestra.run.smithi181.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 591 kB/s | 79 kB 00:00 2024-03-29T04:43:02.201 INFO:teuthology.orchestra.run.smithi181.stdout:(2/2): ceph-fuse-17.2.7-767.g9b4b8ffd.el9.x86_6 1.6 MB/s | 812 kB 00:00 2024-03-29T04:43:02.202 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:43:02.202 INFO:teuthology.orchestra.run.smithi181.stdout:Total 1.8 MB/s | 891 kB 00:00 2024-03-29T04:43:02.202 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:43:02.223 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:43:02.223 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:43:02.296 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:43:02.297 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:43:02.479 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:43:02.613 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-03-29T04:43:02.692 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/2 2024-03-29T04:43:03.364 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/2 2024-03-29T04:43:03.364 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/2 2024-03-29T04:43:03.638 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-03-29T04:43:03.639 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:03.639 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:43:03.639 INFO:teuthology.orchestra.run.smithi181.stdout: ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-03-29T04:43:03.639 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:03.639 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:43:03.952 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install librados-devel 2024-03-29T04:43:04.525 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:08:34 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:43:04.677 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:43:04.677 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:04.678 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repo Size 2024-03-29T04:43:04.678 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:04.678 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:43:04.678 INFO:teuthology.orchestra.run.smithi181.stdout: librados-devel x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 126 k 2024-03-29T04:43:04.678 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:04.678 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:43:04.678 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:04.678 INFO:teuthology.orchestra.run.smithi181.stdout:Install 1 Package 2024-03-29T04:43:04.678 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:04.678 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 126 k 2024-03-29T04:43:04.678 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 448 k 2024-03-29T04:43:04.678 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:43:04.991 INFO:teuthology.orchestra.run.smithi181.stdout:librados-devel-17.2.7-767.g9b4b8ffd.el9.x86_64. 403 kB/s | 126 kB 00:00 2024-03-29T04:43:04.992 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:43:04.992 INFO:teuthology.orchestra.run.smithi181.stdout:Total 401 kB/s | 126 kB 00:00 2024-03-29T04:43:04.992 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:43:04.998 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:43:04.998 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:43:05.052 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:43:05.053 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:43:05.202 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:43:05.305 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:43:05.736 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:43:06.007 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:43:06.007 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:06.007 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:43:06.008 INFO:teuthology.orchestra.run.smithi181.stdout: librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:43:06.008 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:06.008 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:43:06.267 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install libcephfs2 2024-03-29T04:43:06.843 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:08:36 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:43:06.945 INFO:teuthology.orchestra.run.smithi181.stdout:Package libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:43:06.998 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:43:07.000 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2024-03-29T04:43:07.000 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:43:07.070 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install libcephfs-devel 2024-03-29T04:43:07.630 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:08:37 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:43:07.778 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:43:07.779 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:07.779 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repo Size 2024-03-29T04:43:07.779 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:07.780 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:43:07.780 INFO:teuthology.orchestra.run.smithi181.stdout: libcephfs-devel x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 25 k 2024-03-29T04:43:07.780 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:07.780 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:43:07.780 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:07.780 INFO:teuthology.orchestra.run.smithi181.stdout:Install 1 Package 2024-03-29T04:43:07.780 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:07.780 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 25 k 2024-03-29T04:43:07.780 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 110 k 2024-03-29T04:43:07.780 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:43:08.028 INFO:teuthology.orchestra.run.smithi181.stdout:libcephfs-devel-17.2.7-767.g9b4b8ffd.el9.x86_64 101 kB/s | 25 kB 00:00 2024-03-29T04:43:08.028 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:43:08.028 INFO:teuthology.orchestra.run.smithi181.stdout:Total 100 kB/s | 25 kB 00:00 2024-03-29T04:43:08.028 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:43:08.032 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:43:08.032 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:43:08.034 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:43:08.034 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:43:08.123 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:43:08.235 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:43:08.584 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:43:08.829 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:43:08.829 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:08.830 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:43:08.830 INFO:teuthology.orchestra.run.smithi181.stdout: libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:43:08.830 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:08.830 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:43:08.991 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install librados2 2024-03-29T04:43:09.550 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:08:39 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:43:09.648 INFO:teuthology.orchestra.run.smithi181.stdout:Package librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:43:09.700 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:43:09.701 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2024-03-29T04:43:09.701 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:43:09.787 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install librbd1 2024-03-29T04:43:10.350 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:08:40 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:43:10.449 INFO:teuthology.orchestra.run.smithi181.stdout:Package librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:43:10.501 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:43:10.502 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2024-03-29T04:43:10.502 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:43:10.581 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install python3-rados 2024-03-29T04:43:11.145 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:08:41 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:43:11.244 INFO:teuthology.orchestra.run.smithi181.stdout:Package python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:43:11.297 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:43:11.298 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2024-03-29T04:43:11.298 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:43:11.370 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install python3-rgw 2024-03-29T04:43:11.929 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:08:41 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:43:12.029 INFO:teuthology.orchestra.run.smithi181.stdout:Package python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:43:12.080 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:43:12.081 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2024-03-29T04:43:12.081 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:43:12.162 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install python3-cephfs 2024-03-29T04:43:12.720 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:08:42 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:43:12.817 INFO:teuthology.orchestra.run.smithi181.stdout:Package python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:43:12.869 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:43:12.870 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2024-03-29T04:43:12.871 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:43:12.944 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install python3-rbd 2024-03-29T04:43:13.508 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:08:43 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:43:13.605 INFO:teuthology.orchestra.run.smithi181.stdout:Package python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:43:13.658 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:43:13.659 INFO:teuthology.orchestra.run.smithi181.stdout:Nothing to do. 2024-03-29T04:43:13.659 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:43:13.728 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install rbd-fuse 2024-03-29T04:43:14.287 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:08:44 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:43:14.437 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:43:14.438 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:14.438 INFO:teuthology.orchestra.run.smithi181.stdout: Package Architecture Version Repository Size 2024-03-29T04:43:14.438 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:14.438 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:43:14.438 INFO:teuthology.orchestra.run.smithi181.stdout: rbd-fuse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 82 k 2024-03-29T04:43:14.438 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:14.438 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:43:14.438 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:14.438 INFO:teuthology.orchestra.run.smithi181.stdout:Install 1 Package 2024-03-29T04:43:14.438 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:14.438 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 82 k 2024-03-29T04:43:14.439 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 222 k 2024-03-29T04:43:14.439 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:43:14.764 INFO:teuthology.orchestra.run.smithi181.stdout:rbd-fuse-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 252 kB/s | 82 kB 00:00 2024-03-29T04:43:14.765 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:43:14.765 INFO:teuthology.orchestra.run.smithi181.stdout:Total 251 kB/s | 82 kB 00:00 2024-03-29T04:43:14.765 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:43:14.772 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:43:14.772 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:43:14.827 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:43:14.828 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:43:14.962 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:43:15.062 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:43:15.410 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:43:15.718 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:43:15.719 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:15.719 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:43:15.719 INFO:teuthology.orchestra.run.smithi181.stdout: rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:43:15.719 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:15.719 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:43:15.957 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install rbd-mirror 2024-03-29T04:43:16.531 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:08:46 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:43:16.687 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:43:16.688 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:16.688 INFO:teuthology.orchestra.run.smithi181.stdout: Package Arch Version Repo Size 2024-03-29T04:43:16.688 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:16.688 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:43:16.688 INFO:teuthology.orchestra.run.smithi181.stdout: rbd-mirror x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.9 M 2024-03-29T04:43:16.688 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:16.688 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:43:16.688 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:16.688 INFO:teuthology.orchestra.run.smithi181.stdout:Install 1 Package 2024-03-29T04:43:16.689 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:16.689 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 2.9 M 2024-03-29T04:43:16.689 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 12 M 2024-03-29T04:43:16.689 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:43:17.159 INFO:teuthology.orchestra.run.smithi181.stdout:rbd-mirror-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 6.3 MB/s | 2.9 MB 00:00 2024-03-29T04:43:17.160 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:43:17.160 INFO:teuthology.orchestra.run.smithi181.stdout:Total 6.2 MB/s | 2.9 MB 00:00 2024-03-29T04:43:17.160 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:43:17.169 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:43:17.169 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:43:17.241 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:43:17.242 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:43:17.646 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:43:17.710 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:43:17.735 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:43:17.735 INFO:teuthology.orchestra.run.smithi181.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:43:17.735 INFO:teuthology.orchestra.run.smithi181.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-03-29T04:43:17.736 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-29T04:43:17.736 INFO:teuthology.orchestra.run.smithi181.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-29T04:43:17.736 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:18.832 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:43:18.832 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:18.832 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:43:18.832 INFO:teuthology.orchestra.run.smithi181.stdout: rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:43:18.832 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:18.832 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:43:19.079 DEBUG:teuthology.orchestra.run.smithi181:> sudo yum -y install rbd-nbd 2024-03-29T04:43:19.692 INFO:teuthology.orchestra.run.smithi181.stdout:Last metadata expiration check: 0:08:49 ago on Fri 29 Mar 2024 04:34:30 AM UTC. 2024-03-29T04:43:19.842 INFO:teuthology.orchestra.run.smithi181.stdout:Dependencies resolved. 2024-03-29T04:43:19.842 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:19.843 INFO:teuthology.orchestra.run.smithi181.stdout: Package Architecture Version Repository Size 2024-03-29T04:43:19.843 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:19.843 INFO:teuthology.orchestra.run.smithi181.stdout:Installing: 2024-03-29T04:43:19.843 INFO:teuthology.orchestra.run.smithi181.stdout: rbd-nbd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 165 k 2024-03-29T04:43:19.843 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:19.843 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction Summary 2024-03-29T04:43:19.843 INFO:teuthology.orchestra.run.smithi181.stdout:================================================================================ 2024-03-29T04:43:19.843 INFO:teuthology.orchestra.run.smithi181.stdout:Install 1 Package 2024-03-29T04:43:19.843 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:19.843 INFO:teuthology.orchestra.run.smithi181.stdout:Total download size: 165 k 2024-03-29T04:43:19.843 INFO:teuthology.orchestra.run.smithi181.stdout:Installed size: 477 k 2024-03-29T04:43:19.843 INFO:teuthology.orchestra.run.smithi181.stdout:Downloading Packages: 2024-03-29T04:43:20.272 INFO:teuthology.orchestra.run.smithi181.stdout:rbd-nbd-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 386 kB/s | 165 kB 00:00 2024-03-29T04:43:20.272 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:43:20.272 INFO:teuthology.orchestra.run.smithi181.stdout:Total 385 kB/s | 165 kB 00:00 2024-03-29T04:43:20.273 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction check 2024-03-29T04:43:20.280 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction check succeeded. 2024-03-29T04:43:20.280 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction test 2024-03-29T04:43:20.336 INFO:teuthology.orchestra.run.smithi181.stdout:Transaction test succeeded. 2024-03-29T04:43:20.336 INFO:teuthology.orchestra.run.smithi181.stdout:Running transaction 2024-03-29T04:43:20.479 INFO:teuthology.orchestra.run.smithi181.stdout: Preparing : 1/1 2024-03-29T04:43:20.587 INFO:teuthology.orchestra.run.smithi181.stdout: Installing : rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:43:21.025 INFO:teuthology.orchestra.run.smithi181.stdout: Running scriptlet: rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:43:21.296 INFO:teuthology.orchestra.run.smithi181.stdout: Verifying : rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:43:21.296 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:21.296 INFO:teuthology.orchestra.run.smithi181.stdout:Installed: 2024-03-29T04:43:21.296 INFO:teuthology.orchestra.run.smithi181.stdout: rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:43:21.296 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:43:21.296 INFO:teuthology.orchestra.run.smithi181.stdout:Complete! 2024-03-29T04:43:21.567 DEBUG:teuthology.parallel:result is None 2024-03-29T04:43:21.568 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:43:21.568 INFO:teuthology.packaging:ref: None 2024-03-29T04:43:21.568 INFO:teuthology.packaging:tag: None 2024-03-29T04:43:21.568 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:43:21.568 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:43:21.568 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-03-29T04:43:21.908 DEBUG:teuthology.orchestra.run.smithi181:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-03-29T04:43:21.929 INFO:teuthology.orchestra.run.smithi181.stdout:17.2.7-767.g9b4b8ffd.el9 2024-03-29T04:43:21.929 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-767.g9b4b8ffd.el9 2024-03-29T04:43:21.930 INFO:teuthology.task.install:The correct ceph version 17.2.7-767.g9b4b8ffd is installed. 2024-03-29T04:43:21.932 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:43:21.932 INFO:teuthology.packaging:ref: None 2024-03-29T04:43:21.932 INFO:teuthology.packaging:tag: None 2024-03-29T04:43:21.932 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:43:21.932 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:43:21.932 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-03-29T04:43:22.336 DEBUG:teuthology.orchestra.run.smithi187:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-03-29T04:43:22.363 INFO:teuthology.orchestra.run.smithi187.stdout:17.2.7-767.g9b4b8ffd.el9 2024-03-29T04:43:22.363 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-767.g9b4b8ffd.el9 2024-03-29T04:43:22.363 INFO:teuthology.task.install:The correct ceph version 17.2.7-767.g9b4b8ffd is installed. 2024-03-29T04:43:22.365 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-03-29T04:43:22.365 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-03-29T04:43:22.365 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-29T04:43:22.396 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-29T04:43:22.397 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-29T04:43:22.437 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-03-29T04:43:22.437 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-03-29T04:43:22.437 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/usr/bin/daemon-helper 2024-03-29T04:43:22.467 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-29T04:43:22.536 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-29T04:43:22.536 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd of=/usr/bin/daemon-helper 2024-03-29T04:43:22.566 DEBUG:teuthology.orchestra.run.smithi187:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-29T04:43:22.635 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-03-29T04:43:22.635 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-03-29T04:43:22.635 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-29T04:43:22.664 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-29T04:43:22.732 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-29T04:43:22.733 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-29T04:43:22.761 DEBUG:teuthology.orchestra.run.smithi187:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-29T04:43:22.830 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-03-29T04:43:22.830 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-03-29T04:43:22.830 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/usr/bin/stdin-killer 2024-03-29T04:43:22.858 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-29T04:43:22.927 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-29T04:43:22.927 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd of=/usr/bin/stdin-killer 2024-03-29T04:43:22.955 DEBUG:teuthology.orchestra.run.smithi187:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-29T04:43:23.023 INFO:teuthology.run_tasks:Running task cephadm... 2024-03-29T04:43:23.132 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '8da23624bc17fe0b1f797227e72a0db0ce7821ba'} 2024-03-29T04:43:23.133 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-03-29T04:43:23.133 INFO:tasks.cephadm:Cluster fsid is dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:43:23.133 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-03-29T04:43:23.134 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.181', 'mon.c': '[v2:172.21.15.181:3301,v1:172.21.15.181:6790]', 'mon.b': '172.21.15.187'} 2024-03-29T04:43:23.134 INFO:tasks.cephadm:First mon is mon.a on smithi181 2024-03-29T04:43:23.134 INFO:tasks.cephadm:First mgr is y 2024-03-29T04:43:23.134 INFO:tasks.cephadm:Normalizing hostnames... 2024-03-29T04:43:23.134 DEBUG:teuthology.orchestra.run.smithi181:> sudo hostname $(hostname -s) 2024-03-29T04:43:23.163 DEBUG:teuthology.orchestra.run.smithi187:> sudo hostname $(hostname -s) 2024-03-29T04:43:23.192 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-03-29T04:43:23.192 DEBUG:teuthology.orchestra.run.smithi181:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-29T04:43:23.453 INFO:teuthology.orchestra.run.smithi181.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Mar 29 04:43 /home/ubuntu/cephtest/cephadm 2024-03-29T04:43:23.454 DEBUG:teuthology.orchestra.run.smithi187:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-29T04:43:23.624 INFO:teuthology.orchestra.run.smithi187.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Mar 29 04:43 /home/ubuntu/cephtest/cephadm 2024-03-29T04:43:23.624 DEBUG:teuthology.orchestra.run.smithi181:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-29T04:43:23.641 DEBUG:teuthology.orchestra.run.smithi187:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-29T04:43:23.667 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-03-29T04:43:23.667 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-03-29T04:43:23.684 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-03-29T04:43:24.515 INFO:teuthology.orchestra.run.smithi187.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-03-29T04:43:24.541 INFO:teuthology.orchestra.run.smithi181.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-03-29T04:44:31.557 INFO:teuthology.orchestra.run.smithi187.stdout:{ 2024-03-29T04:44:31.558 INFO:teuthology.orchestra.run.smithi187.stdout: "ceph_version": "ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)", 2024-03-29T04:44:31.558 INFO:teuthology.orchestra.run.smithi187.stdout: "image_id": "cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a", 2024-03-29T04:44:31.558 INFO:teuthology.orchestra.run.smithi187.stdout: "repo_digests": [ 2024-03-29T04:44:31.558 INFO:teuthology.orchestra.run.smithi187.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b" 2024-03-29T04:44:31.558 INFO:teuthology.orchestra.run.smithi187.stdout: ] 2024-03-29T04:44:31.558 INFO:teuthology.orchestra.run.smithi187.stdout:} 2024-03-29T04:44:39.702 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2024-03-29T04:44:39.703 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph_version": "ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)", 2024-03-29T04:44:39.703 INFO:teuthology.orchestra.run.smithi181.stdout: "image_id": "cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a", 2024-03-29T04:44:39.703 INFO:teuthology.orchestra.run.smithi181.stdout: "repo_digests": [ 2024-03-29T04:44:39.703 INFO:teuthology.orchestra.run.smithi181.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b" 2024-03-29T04:44:39.703 INFO:teuthology.orchestra.run.smithi181.stdout: ] 2024-03-29T04:44:39.703 INFO:teuthology.orchestra.run.smithi181.stdout:} 2024-03-29T04:44:39.721 DEBUG:teuthology.orchestra.run.smithi181:> sudo mkdir -p /etc/ceph 2024-03-29T04:44:39.778 DEBUG:teuthology.orchestra.run.smithi187:> sudo mkdir -p /etc/ceph 2024-03-29T04:44:39.813 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 777 /etc/ceph 2024-03-29T04:44:39.840 DEBUG:teuthology.orchestra.run.smithi187:> sudo chmod 777 /etc/ceph 2024-03-29T04:44:39.883 INFO:tasks.cephadm:Writing seed config... 2024-03-29T04:44:39.884 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-03-29T04:44:39.884 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-03-29T04:44:39.884 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-03-29T04:44:39.884 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-03-29T04:44:39.884 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-03-29T04:44:39.885 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-03-29T04:44:39.885 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-03-29T04:44:39.885 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-03-29T04:44:39.885 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-03-29T04:44:39.885 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-03-29T04:44:39.885 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-03-29T04:44:39.885 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-03-29T04:44:39.885 DEBUG:teuthology.orchestra.run.smithi181:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-03-29T04:44:39.900 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = dfcecab8-ed86-11ee-95d0-87774f69a715 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-03-29T04:44:39.900 DEBUG:teuthology.orchestra.run.smithi181:mon.a> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.a.service 2024-03-29T04:44:39.943 DEBUG:teuthology.orchestra.run.smithi181:mgr.y> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.y.service 2024-03-29T04:44:39.985 INFO:tasks.cephadm:Bootstrapping... 2024-03-29T04:44:39.986 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.181 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:44:40.189 INFO:teuthology.orchestra.run.smithi181.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:44:40.189 INFO:teuthology.orchestra.run.smithi181.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'dfcecab8-ed86-11ee-95d0-87774f69a715', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.181', '--skip-admin-label'] 2024-03-29T04:44:40.215 INFO:teuthology.orchestra.run.smithi181.stdout:/bin/podman: stdout 4.9.4-dev 2024-03-29T04:44:40.216 INFO:teuthology.orchestra.run.smithi181.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-03-29T04:44:40.216 INFO:teuthology.orchestra.run.smithi181.stdout:Verifying podman|docker is present... 2024-03-29T04:44:40.239 INFO:teuthology.orchestra.run.smithi181.stdout:/bin/podman: stdout 4.9.4-dev 2024-03-29T04:44:40.239 INFO:teuthology.orchestra.run.smithi181.stdout:Verifying lvm2 is present... 2024-03-29T04:44:40.239 INFO:teuthology.orchestra.run.smithi181.stdout:Verifying time synchronization is in place... 2024-03-29T04:44:40.247 INFO:teuthology.orchestra.run.smithi181.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-03-29T04:44:40.247 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-03-29T04:44:40.253 INFO:teuthology.orchestra.run.smithi181.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-03-29T04:44:40.253 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stdout inactive 2024-03-29T04:44:40.260 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stdout enabled 2024-03-29T04:44:40.266 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stdout active 2024-03-29T04:44:40.266 INFO:teuthology.orchestra.run.smithi181.stdout:Unit chronyd.service is enabled and running 2024-03-29T04:44:40.266 INFO:teuthology.orchestra.run.smithi181.stdout:Repeating the final host check... 2024-03-29T04:44:40.289 INFO:teuthology.orchestra.run.smithi181.stdout:/bin/podman: stdout 4.9.4-dev 2024-03-29T04:44:40.289 INFO:teuthology.orchestra.run.smithi181.stdout:podman (/bin/podman) version 4.9.4 is present 2024-03-29T04:44:40.289 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl is present 2024-03-29T04:44:40.289 INFO:teuthology.orchestra.run.smithi181.stdout:lvcreate is present 2024-03-29T04:44:40.295 INFO:teuthology.orchestra.run.smithi181.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-03-29T04:44:40.295 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-03-29T04:44:40.302 INFO:teuthology.orchestra.run.smithi181.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-03-29T04:44:40.302 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stdout inactive 2024-03-29T04:44:40.309 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stdout enabled 2024-03-29T04:44:40.315 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stdout active 2024-03-29T04:44:40.315 INFO:teuthology.orchestra.run.smithi181.stdout:Unit chronyd.service is enabled and running 2024-03-29T04:44:40.315 INFO:teuthology.orchestra.run.smithi181.stdout:Host looks OK 2024-03-29T04:44:40.315 INFO:teuthology.orchestra.run.smithi181.stdout:Cluster fsid: dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:44:40.315 INFO:teuthology.orchestra.run.smithi181.stdout:Acquiring lock 140469298127968 on /run/cephadm/dfcecab8-ed86-11ee-95d0-87774f69a715.lock 2024-03-29T04:44:40.315 INFO:teuthology.orchestra.run.smithi181.stdout:Lock 140469298127968 acquired on /run/cephadm/dfcecab8-ed86-11ee-95d0-87774f69a715.lock 2024-03-29T04:44:40.316 INFO:teuthology.orchestra.run.smithi181.stdout:Verifying IP 172.21.15.181 port 3300 ... 2024-03-29T04:44:40.316 INFO:teuthology.orchestra.run.smithi181.stdout:Verifying IP 172.21.15.181 port 6789 ... 2024-03-29T04:44:40.316 INFO:teuthology.orchestra.run.smithi181.stdout:Base mon IP(s) is [172.21.15.181:3300, 172.21.15.181:6789], mon addrv is [v2:172.21.15.181:3300,v1:172.21.15.181:6789] 2024-03-29T04:44:40.319 INFO:teuthology.orchestra.run.smithi181.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.181 metric 100 2024-03-29T04:44:40.320 INFO:teuthology.orchestra.run.smithi181.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.181 metric 100 2024-03-29T04:44:40.323 INFO:teuthology.orchestra.run.smithi181.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-03-29T04:44:40.323 INFO:teuthology.orchestra.run.smithi181.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-03-29T04:44:40.323 INFO:teuthology.orchestra.run.smithi181.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1792sec hoplimit 64 pref medium 2024-03-29T04:44:40.325 INFO:teuthology.orchestra.run.smithi181.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-03-29T04:44:40.326 INFO:teuthology.orchestra.run.smithi181.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-03-29T04:44:40.326 INFO:teuthology.orchestra.run.smithi181.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-03-29T04:44:40.326 INFO:teuthology.orchestra.run.smithi181.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-03-29T04:44:40.326 INFO:teuthology.orchestra.run.smithi181.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe88:7b9d/64 scope link 2024-03-29T04:44:40.326 INFO:teuthology.orchestra.run.smithi181.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-03-29T04:44:40.326 INFO:teuthology.orchestra.run.smithi181.stdout:Mon IP `172.21.15.181` is in CIDR network `172.21.0.0/20` 2024-03-29T04:44:40.326 INFO:teuthology.orchestra.run.smithi181.stdout:Mon IP `172.21.15.181` is in CIDR network `172.21.0.0/20` 2024-03-29T04:44:40.326 INFO:teuthology.orchestra.run.smithi181.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-03-29T04:44:40.327 INFO:teuthology.orchestra.run.smithi181.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-03-29T04:44:40.327 INFO:teuthology.orchestra.run.smithi181.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-03-29T04:44:40.802 INFO:teuthology.orchestra.run.smithi181.stdout:/bin/podman: stdout cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a 2024-03-29T04:44:40.803 INFO:teuthology.orchestra.run.smithi181.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-03-29T04:44:40.803 INFO:teuthology.orchestra.run.smithi181.stdout:/bin/podman: stderr Getting image source signatures 2024-03-29T04:44:40.803 INFO:teuthology.orchestra.run.smithi181.stdout:/bin/podman: stderr Copying blob sha256:7ae200e79e73f55e16b47791a08422a21d8b9f8d5bb767e6ae856e160a74c37a 2024-03-29T04:44:40.803 INFO:teuthology.orchestra.run.smithi181.stdout:/bin/podman: stderr Copying blob sha256:11da15500eca1f826c328eb3265caa30b0a3730510e2a32023084d6fe9cee20c 2024-03-29T04:44:40.803 INFO:teuthology.orchestra.run.smithi181.stdout:/bin/podman: stderr Copying config sha256:cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a 2024-03-29T04:44:40.803 INFO:teuthology.orchestra.run.smithi181.stdout:/bin/podman: stderr Writing manifest to image destination 2024-03-29T04:44:41.740 INFO:teuthology.orchestra.run.smithi181.stdout:ceph: stdout ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable) 2024-03-29T04:44:41.741 INFO:teuthology.orchestra.run.smithi181.stdout:Ceph version: ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable) 2024-03-29T04:44:41.741 INFO:teuthology.orchestra.run.smithi181.stdout:Extracting ceph user uid/gid from container image... 2024-03-29T04:44:42.797 INFO:teuthology.orchestra.run.smithi181.stdout:stat: stdout 167 167 2024-03-29T04:44:42.797 INFO:teuthology.orchestra.run.smithi181.stdout:Creating initial keys... 2024-03-29T04:44:43.710 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph-authtool: stdout AQA7RwZmbVpmChAATdixr9uWzfcFG5+k1JP+GA== 2024-03-29T04:44:44.767 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph-authtool: stdout AQA8RwZmna8+BBAAFMQ0RY2BjALDt2fOHgZgQw== 2024-03-29T04:44:45.766 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph-authtool: stdout AQA9RwZmL0MXBhAAPrdLQ1AfN/dVaNC5c2M3NA== 2024-03-29T04:44:45.766 INFO:teuthology.orchestra.run.smithi181.stdout:Creating initial monmap... 2024-03-29T04:44:46.746 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-29T04:44:46.747 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-03-29T04:44:46.747 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:44:46.747 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-29T04:44:46.747 INFO:teuthology.orchestra.run.smithi181.stdout:monmaptool for a [v2:172.21.15.181:3300,v1:172.21.15.181:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-29T04:44:46.747 INFO:teuthology.orchestra.run.smithi181.stdout:setting min_mon_release = octopus 2024-03-29T04:44:46.748 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/monmaptool: set fsid to dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:44:46.748 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-29T04:44:46.748 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:44:46.748 INFO:teuthology.orchestra.run.smithi181.stdout:Creating mon... 2024-03-29T04:44:47.786 INFO:teuthology.orchestra.run.smithi181.stdout:create mon.a on 2024-03-29T04:44:47.979 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-03-29T04:44:48.112 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-03-29T04:44:48.256 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-dfcecab8-ed86-11ee-95d0-87774f69a715.target → /etc/systemd/system/ceph-dfcecab8-ed86-11ee-95d0-87774f69a715.target. 2024-03-29T04:44:48.256 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-dfcecab8-ed86-11ee-95d0-87774f69a715.target → /etc/systemd/system/ceph-dfcecab8-ed86-11ee-95d0-87774f69a715.target. 2024-03-29T04:44:48.416 INFO:teuthology.orchestra.run.smithi181.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.a 2024-03-29T04:44:48.416 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stderr Failed to reset failed state of unit ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.a.service: Unit ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.a.service not loaded. 2024-03-29T04:44:48.581 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-dfcecab8-ed86-11ee-95d0-87774f69a715.target.wants/ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.a.service → /etc/systemd/system/ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@.service. 2024-03-29T04:44:49.139 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:44:49 smithi181 ceph-mon[29288]: mon.a is new leader, mons a in quorum (ranks 0) 2024-03-29T04:44:49.152 INFO:teuthology.orchestra.run.smithi181.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-03-29T04:44:49.152 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stdout disabled 2024-03-29T04:44:49.159 INFO:teuthology.orchestra.run.smithi181.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-03-29T04:44:49.159 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stdout inactive 2024-03-29T04:44:49.159 INFO:teuthology.orchestra.run.smithi181.stdout:firewalld.service is not enabled 2024-03-29T04:44:49.160 INFO:teuthology.orchestra.run.smithi181.stdout:Not possible to enable service . firewalld.service is not available 2024-03-29T04:44:49.160 INFO:teuthology.orchestra.run.smithi181.stdout:Waiting for mon to start... 2024-03-29T04:44:49.160 INFO:teuthology.orchestra.run.smithi181.stdout:Waiting for mon... 2024-03-29T04:44:49.445 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:44:49 smithi181 bash[29250]: ed4cb388f0872007abb112daacabe0f1769fb3678a3aeb444ea01a0e6377f9e2 2024-03-29T04:44:49.445 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:44:49 smithi181 systemd[1]: Started Ceph mon.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:44:50.445 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:44:50 smithi181 ceph-mon[29288]: mon.a is new leader, mons a in quorum (ranks 0) 2024-03-29T04:44:50.445 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:44:50 smithi181 ceph-mon[29288]: monmap e1: 1 mons at {a=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:44:50.445 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:44:50 smithi181 ceph-mon[29288]: fsmap 2024-03-29T04:44:50.445 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:44:50 smithi181 ceph-mon[29288]: osdmap e1: 0 total, 0 up, 0 in 2024-03-29T04:44:50.445 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:44:50 smithi181 ceph-mon[29288]: mgrmap e1: no daemons active 2024-03-29T04:44:52.694 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:44:52 smithi181 ceph-mon[29288]: from='client.? 172.21.15.181:0/289735245' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-03-29T04:44:58.638 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout cluster: 2024-03-29T04:44:58.638 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout id: dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:44:58.638 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-03-29T04:44:58.639 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 2024-03-29T04:44:58.639 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout services: 2024-03-29T04:44:58.639 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 3s) 2024-03-29T04:44:58.639 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-03-29T04:44:58.639 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-03-29T04:44:58.639 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 2024-03-29T04:44:58.639 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout data: 2024-03-29T04:44:58.640 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-03-29T04:44:58.640 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-03-29T04:44:58.640 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-03-29T04:44:58.640 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout pgs: 2024-03-29T04:44:58.640 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 2024-03-29T04:44:58.640 INFO:teuthology.orchestra.run.smithi181.stdout:mon is available 2024-03-29T04:44:58.640 INFO:teuthology.orchestra.run.smithi181.stdout:Assimilating anything we can from ceph.conf... 2024-03-29T04:44:59.445 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:44:59 smithi181 ceph-mon[29288]: from='client.? 172.21.15.181:0/1032149516' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-29T04:44:59.445 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:44:59 smithi181 ceph-mon[29288]: from='client.? 172.21.15.181:0/1032149516' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-03-29T04:44:59.719 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 2024-03-29T04:44:59.719 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout [global] 2024-03-29T04:44:59.719 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout fsid = dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:44:59.720 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.181:3300,v1:172.21.15.181:6789] 2024-03-29T04:44:59.720 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-03-29T04:44:59.720 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-03-29T04:44:59.720 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-03-29T04:44:59.720 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-03-29T04:44:59.720 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 2024-03-29T04:44:59.720 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout [mgr] 2024-03-29T04:44:59.720 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-03-29T04:44:59.720 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 2024-03-29T04:44:59.720 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout [osd] 2024-03-29T04:44:59.720 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-03-29T04:44:59.720 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-03-29T04:44:59.720 INFO:teuthology.orchestra.run.smithi181.stdout:Generating new minimal ceph.conf... 2024-03-29T04:45:00.444 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:00 smithi181 ceph-mon[29288]: from='client.? 172.21.15.181:0/4102776529' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:00.766 INFO:teuthology.orchestra.run.smithi181.stdout:Restarting the monitor... 2024-03-29T04:45:01.045 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:00 smithi181 systemd[1]: Stopping Ceph mon.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:45:01.045 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:01 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a[29265]: 2024-03-29T04:45:01.044+0000 7f7d6acc0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:45:01.045 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:01 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a[29265]: 2024-03-29T04:45:01.044+0000 7f7d6acc0700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-03-29T04:45:01.441 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:01 smithi181 podman[29610]: 2024-03-29 04:45:01.23961573 +0000 UTC m=+0.303992235 container died ed4cb388f0872007abb112daacabe0f1769fb3678a3aeb444ea01a0e6377f9e2 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-4213e17) 2024-03-29T04:45:01.695 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:01 smithi181 podman[29610]: 2024-03-29 04:45:01.533791939 +0000 UTC m=+0.598168444 container cleanup ed4cb388f0872007abb112daacabe0f1769fb3678a3aeb444ea01a0e6377f9e2 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0) 2024-03-29T04:45:01.695 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:01 smithi181 bash[29610]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a 2024-03-29T04:45:02.151 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:01 smithi181 podman[29629]: 2024-03-29 04:45:01.809999518 +0000 UTC m=+0.561853783 container remove ed4cb388f0872007abb112daacabe0f1769fb3678a3aeb444ea01a0e6377f9e2 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a, ceph=True, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-03-29T04:45:02.151 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:01 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.a.service: Deactivated successfully. 2024-03-29T04:45:02.151 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:01 smithi181 systemd[1]: Stopped Ceph mon.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:45:02.151 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:01 smithi181 systemd[1]: Starting Ceph mon.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:45:02.412 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 podman[29727]: 2024-03-29 04:45:02.152543403 +0000 UTC m=+0.105791081 container create 3e3936015cad924909bce492f26f753eb72f818dfd57ce0d65915978f085b479 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=quincy-4213e17, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-03-29T04:45:02.412 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 podman[29727]: 2024-03-29 04:45:02.064692762 +0000 UTC m=+0.017940408 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph:quincy 2024-03-29T04:45:02.412 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 podman[29727]: 2024-03-29 04:45:02.329562272 +0000 UTC m=+0.282809915 container init 3e3936015cad924909bce492f26f753eb72f818dfd57ce0d65915978f085b479 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a, org.label-schema.license=GPLv2, RELEASE=quincy-4213e17, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True) 2024-03-29T04:45:02.412 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 podman[29727]: 2024-03-29 04:45:02.335534492 +0000 UTC m=+0.288782126 container start 3e3936015cad924909bce492f26f753eb72f818dfd57ce0d65915978f085b479 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, io.buildah.version=1.33.5) 2024-03-29T04:45:02.412 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: set uid:gid to 167:167 (ceph:ceph) 2024-03-29T04:45:02.412 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable), process ceph-mon, pid 7 2024-03-29T04:45:02.412 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: pidfile_write: ignore empty --pid-file 2024-03-29T04:45:02.412 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: load: jerasure load: lrc 2024-03-29T04:45:02.412 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: RocksDB version: 6.15.5 2024-03-29T04:45:02.412 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-03-29T04:45:02.412 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Compile date Mar 28 2024 2024-03-29T04:45:02.413 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: DB SUMMARY 2024-03-29T04:45:02.413 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: DB Session ID: DFFU8Y4S52TV7713NFC7 2024-03-29T04:45:02.413 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: CURRENT file: CURRENT 2024-03-29T04:45:02.413 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: IDENTITY file: IDENTITY 2024-03-29T04:45:02.413 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-03-29T04:45:02.413 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-03-29T04:45:02.413 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 84538 ; 2024-03-29T04:45:02.413 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.error_if_exists: 0 2024-03-29T04:45:02.413 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.create_if_missing: 0 2024-03-29T04:45:02.413 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.paranoid_checks: 1 2024-03-29T04:45:02.413 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-29T04:45:02.413 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.env: 0x55c111deea80 2024-03-29T04:45:02.413 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.fs: Posix File System 2024-03-29T04:45:02.413 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.info_log: 0x55c1126cce80 2024-03-29T04:45:02.414 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_file_opening_threads: 16 2024-03-29T04:45:02.414 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.statistics: (nil) 2024-03-29T04:45:02.414 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.use_fsync: 0 2024-03-29T04:45:02.414 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_log_file_size: 0 2024-03-29T04:45:02.414 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-29T04:45:02.414 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-29T04:45:02.414 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.keep_log_file_num: 1000 2024-03-29T04:45:02.414 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.recycle_log_file_num: 0 2024-03-29T04:45:02.414 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.allow_fallocate: 1 2024-03-29T04:45:02.414 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.allow_mmap_reads: 0 2024-03-29T04:45:02.414 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.allow_mmap_writes: 0 2024-03-29T04:45:02.415 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.use_direct_reads: 0 2024-03-29T04:45:02.415 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-29T04:45:02.415 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.create_missing_column_families: 0 2024-03-29T04:45:02.415 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.db_log_dir: 2024-03-29T04:45:02.415 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-03-29T04:45:02.415 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-29T04:45:02.415 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-29T04:45:02.415 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-29T04:45:02.415 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-29T04:45:02.415 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-29T04:45:02.415 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-29T04:45:02.415 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.advise_random_on_open: 1 2024-03-29T04:45:02.415 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.db_write_buffer_size: 0 2024-03-29T04:45:02.415 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.write_buffer_manager: 0x55c11343c900 2024-03-29T04:45:02.415 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-29T04:45:02.416 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-03-29T04:45:02.416 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-29T04:45:02.416 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-29T04:45:02.416 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.rate_limiter: (nil) 2024-03-29T04:45:02.416 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-29T04:45:02.416 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.wal_recovery_mode: 2 2024-03-29T04:45:02.416 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.enable_thread_tracking: 0 2024-03-29T04:45:02.416 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.enable_pipelined_write: 0 2024-03-29T04:45:02.417 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.unordered_write: 0 2024-03-29T04:45:02.418 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-29T04:45:02.418 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-29T04:45:02.418 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-29T04:45:02.418 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-29T04:45:02.418 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.row_cache: None 2024-03-29T04:45:02.418 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.wal_filter: None 2024-03-29T04:45:02.418 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-29T04:45:02.418 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.allow_ingest_behind: 0 2024-03-29T04:45:02.418 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.preserve_deletes: 0 2024-03-29T04:45:02.418 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.two_write_queues: 0 2024-03-29T04:45:02.418 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.manual_wal_flush: 0 2024-03-29T04:45:02.419 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.atomic_flush: 0 2024-03-29T04:45:02.419 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-29T04:45:02.419 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-29T04:45:02.419 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-29T04:45:02.419 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.log_readahead_size: 0 2024-03-29T04:45:02.419 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-29T04:45:02.419 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.best_efforts_recovery: 0 2024-03-29T04:45:02.419 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-29T04:45:02.419 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-29T04:45:02.419 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.allow_data_in_errors: 0 2024-03-29T04:45:02.419 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.db_host_id: __hostname__ 2024-03-29T04:45:02.419 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_background_jobs: 2 2024-03-29T04:45:02.419 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_background_compactions: -1 2024-03-29T04:45:02.420 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_subcompactions: 1 2024-03-29T04:45:02.420 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-29T04:45:02.420 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-29T04:45:02.420 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-29T04:45:02.420 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_total_wal_size: 0 2024-03-29T04:45:02.420 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-29T04:45:02.420 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-29T04:45:02.420 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-29T04:45:02.420 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-29T04:45:02.420 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_open_files: -1 2024-03-29T04:45:02.420 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.bytes_per_sync: 0 2024-03-29T04:45:02.420 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-29T04:45:02.420 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-29T04:45:02.420 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compaction_readahead_size: 0 2024-03-29T04:45:02.421 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_background_flushes: -1 2024-03-29T04:45:02.421 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Compression algorithms supported: 2024-03-29T04:45:02.421 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-29T04:45:02.421 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: kZSTD supported: 0 2024-03-29T04:45:02.421 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: kXpressCompression supported: 0 2024-03-29T04:45:02.421 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: kLZ4HCCompression supported: 1 2024-03-29T04:45:02.421 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: kZlibCompression supported: 1 2024-03-29T04:45:02.421 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: kSnappyCompression supported: 1 2024-03-29T04:45:02.421 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: kLZ4Compression supported: 1 2024-03-29T04:45:02.421 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: kBZip2Compression supported: 0 2024-03-29T04:45:02.421 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-29T04:45:02.421 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-03-29T04:45:02.421 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-03-29T04:45:02.421 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-29T04:45:02.422 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.merge_operator: 2024-03-29T04:45:02.422 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compaction_filter: None 2024-03-29T04:45:02.422 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compaction_filter_factory: None 2024-03-29T04:45:02.422 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.sst_partitioner_factory: None 2024-03-29T04:45:02.422 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-29T04:45:02.422 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-29T04:45:02.422 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c11271cf58) 2024-03-29T04:45:02.422 INFO:journalctl@ceph.mon.a.smithi181.stdout: cache_index_and_filter_blocks: 1 2024-03-29T04:45:02.422 INFO:journalctl@ceph.mon.a.smithi181.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-29T04:45:02.422 INFO:journalctl@ceph.mon.a.smithi181.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-29T04:45:02.422 INFO:journalctl@ceph.mon.a.smithi181.stdout: pin_top_level_index_and_filter: 1 2024-03-29T04:45:02.422 INFO:journalctl@ceph.mon.a.smithi181.stdout: index_type: 0 2024-03-29T04:45:02.423 INFO:journalctl@ceph.mon.a.smithi181.stdout: data_block_index_type: 0 2024-03-29T04:45:02.423 INFO:journalctl@ceph.mon.a.smithi181.stdout: index_shortening: 1 2024-03-29T04:45:02.423 INFO:journalctl@ceph.mon.a.smithi181.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-29T04:45:02.423 INFO:journalctl@ceph.mon.a.smithi181.stdout: hash_index_allow_collision: 1 2024-03-29T04:45:02.423 INFO:journalctl@ceph.mon.a.smithi181.stdout: checksum: 1 2024-03-29T04:45:02.423 INFO:journalctl@ceph.mon.a.smithi181.stdout: no_block_cache: 0 2024-03-29T04:45:02.423 INFO:journalctl@ceph.mon.a.smithi181.stdout: block_cache: 0x55c11274f090 2024-03-29T04:45:02.423 INFO:journalctl@ceph.mon.a.smithi181.stdout: block_cache_name: BinnedLRUCache 2024-03-29T04:45:02.423 INFO:journalctl@ceph.mon.a.smithi181.stdout: block_cache_options: 2024-03-29T04:45:02.423 INFO:journalctl@ceph.mon.a.smithi181.stdout: capacity : 536870912 2024-03-29T04:45:02.423 INFO:journalctl@ceph.mon.a.smithi181.stdout: num_shard_bits : 4 2024-03-29T04:45:02.423 INFO:journalctl@ceph.mon.a.smithi181.stdout: strict_capacity_limit : 0 2024-03-29T04:45:02.423 INFO:journalctl@ceph.mon.a.smithi181.stdout: high_pri_pool_ratio: 0.000 2024-03-29T04:45:02.423 INFO:journalctl@ceph.mon.a.smithi181.stdout: block_cache_compressed: (nil) 2024-03-29T04:45:02.424 INFO:journalctl@ceph.mon.a.smithi181.stdout: persistent_cache: (nil) 2024-03-29T04:45:02.424 INFO:journalctl@ceph.mon.a.smithi181.stdout: block_size: 4096 2024-03-29T04:45:02.424 INFO:journalctl@ceph.mon.a.smithi181.stdout: block_size_deviation: 10 2024-03-29T04:45:02.424 INFO:journalctl@ceph.mon.a.smithi181.stdout: block_restart_interval: 16 2024-03-29T04:45:02.424 INFO:journalctl@ceph.mon.a.smithi181.stdout: index_block_restart_interval: 1 2024-03-29T04:45:02.424 INFO:journalctl@ceph.mon.a.smithi181.stdout: metadata_block_size: 4096 2024-03-29T04:45:02.424 INFO:journalctl@ceph.mon.a.smithi181.stdout: partition_filters: 0 2024-03-29T04:45:02.424 INFO:journalctl@ceph.mon.a.smithi181.stdout: use_delta_encoding: 1 2024-03-29T04:45:02.424 INFO:journalctl@ceph.mon.a.smithi181.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-03-29T04:45:02.424 INFO:journalctl@ceph.mon.a.smithi181.stdout: whole_key_filtering: 1 2024-03-29T04:45:02.424 INFO:journalctl@ceph.mon.a.smithi181.stdout: verify_compression: 0 2024-03-29T04:45:02.424 INFO:journalctl@ceph.mon.a.smithi181.stdout: read_amp_bytes_per_bit: 0 2024-03-29T04:45:02.424 INFO:journalctl@ceph.mon.a.smithi181.stdout: format_version: 4 2024-03-29T04:45:02.424 INFO:journalctl@ceph.mon.a.smithi181.stdout: enable_index_compression: 1 2024-03-29T04:45:02.424 INFO:journalctl@ceph.mon.a.smithi181.stdout: block_align: 0 2024-03-29T04:45:02.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.write_buffer_size: 33554432 2024-03-29T04:45:02.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_write_buffer_number: 2 2024-03-29T04:45:02.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compression: NoCompression 2024-03-29T04:45:02.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.bottommost_compression: Disabled 2024-03-29T04:45:02.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.prefix_extractor: nullptr 2024-03-29T04:45:02.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-29T04:45:02.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.num_levels: 7 2024-03-29T04:45:02.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-29T04:45:02.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-29T04:45:02.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-29T04:45:02.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-29T04:45:02.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-29T04:45:02.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-29T04:45:02.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-29T04:45:02.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:45:02.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-29T04:45:02.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-29T04:45:02.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-29T04:45:02.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compression_opts.level: 32767 2024-03-29T04:45:02.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compression_opts.strategy: 0 2024-03-29T04:45:02.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-29T04:45:02.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:45:02.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-29T04:45:02.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compression_opts.enabled: false 2024-03-29T04:45:02.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-29T04:45:02.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-29T04:45:02.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-29T04:45:02.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.target_file_size_base: 67108864 2024-03-29T04:45:02.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-29T04:45:02.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-29T04:45:02.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-29T04:45:02.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-29T04:45:02.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-29T04:45:02.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-29T04:45:02.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-29T04:45:02.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-29T04:45:02.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-29T04:45:02.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-29T04:45:02.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-29T04:45:02.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-29T04:45:02.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-29T04:45:02.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.arena_block_size: 4194304 2024-03-29T04:45:02.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-29T04:45:02.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-29T04:45:02.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-03-29T04:45:02.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.disable_auto_compactions: 0 2024-03-29T04:45:02.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-29T04:45:02.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-29T04:45:02.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-29T04:45:02.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-29T04:45:02.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-29T04:45:02.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-29T04:45:02.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-29T04:45:02.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-29T04:45:02.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-29T04:45:02.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-29T04:45:02.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.table_properties_collectors: 2024-03-29T04:45:02.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.inplace_update_support: 0 2024-03-29T04:45:02.429 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-29T04:45:02.429 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-29T04:45:02.429 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-29T04:45:02.429 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-29T04:45:02.429 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.bloom_locality: 0 2024-03-29T04:45:02.429 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.max_successive_merges: 0 2024-03-29T04:45:02.429 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-29T04:45:02.429 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.paranoid_file_checks: 0 2024-03-29T04:45:02.429 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.force_consistency_checks: 1 2024-03-29T04:45:02.429 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.report_bg_io_stats: 0 2024-03-29T04:45:02.429 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.ttl: 2592000 2024-03-29T04:45:02.429 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-29T04:45:02.429 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.enable_blob_files: false 2024-03-29T04:45:02.429 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.min_blob_size: 0 2024-03-29T04:45:02.429 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.blob_file_size: 268435456 2024-03-29T04:45:02.429 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-29T04:45:02.430 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-29T04:45:02.430 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-29T04:45:02.430 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-29T04:45:02.430 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-03-29T04:45:02.430 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-03-29T04:45:02.430 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687502362118, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-03-29T04:45:02.430 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-03-29T04:45:02.430 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-03-29T04:45:02.430 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687502362859, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80822, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 79083, "index_size": 227, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10297, "raw_average_key_size": 47, "raw_value_size": 73266, "raw_average_value_size": 337, "num_data_blocks": 10, "num_entries": 217, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1711687502, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "23916bf6-3b53-4ca2-89d1-ad25f90d246d", "db_session_id": "DFFU8Y4S52TV7713NFC7"}} 2024-03-29T04:45:02.430 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-03-29T04:45:02.430 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687502363194, "job": 1, "event": "recovery_finished"} 2024-03-29T04:45:02.430 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-29T04:45:02.430 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55c1126e3180 2024-03-29T04:45:02.431 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: DB pointer 0x55c1127b8000 2024-03-29T04:45:02.431 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-03-29T04:45:02.431 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-03-29T04:45:02.431 INFO:journalctl@ceph.mon.a.smithi181.stdout: ** DB Stats ** 2024-03-29T04:45:02.431 INFO:journalctl@ceph.mon.a.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:45:02.431 INFO:journalctl@ceph.mon.a.smithi181.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-29T04:45:02.431 INFO:journalctl@ceph.mon.a.smithi181.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:45:02.431 INFO:journalctl@ceph.mon.a.smithi181.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:45:02.431 INFO:journalctl@ceph.mon.a.smithi181.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-29T04:45:02.431 INFO:journalctl@ceph.mon.a.smithi181.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-03-29T04:45:02.431 INFO:journalctl@ceph.mon.a.smithi181.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:45:02.431 INFO:journalctl@ceph.mon.a.smithi181.stdout: 2024-03-29T04:45:02.431 INFO:journalctl@ceph.mon.a.smithi181.stdout: ** Compaction Stats [default] ** 2024-03-29T04:45:02.431 INFO:journalctl@ceph.mon.a.smithi181.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:45:02.431 INFO:journalctl@ceph.mon.a.smithi181.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:45:02.432 INFO:journalctl@ceph.mon.a.smithi181.stdout: L0 2/0 80.69 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.7 0.00 0.00 1 0.001 0 0 2024-03-29T04:45:02.432 INFO:journalctl@ceph.mon.a.smithi181.stdout: Sum 2/0 80.69 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.7 0.00 0.00 1 0.001 0 0 2024-03-29T04:45:02.432 INFO:journalctl@ceph.mon.a.smithi181.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.7 0.00 0.00 1 0.001 0 0 2024-03-29T04:45:02.432 INFO:journalctl@ceph.mon.a.smithi181.stdout: 2024-03-29T04:45:02.432 INFO:journalctl@ceph.mon.a.smithi181.stdout: ** Compaction Stats [default] ** 2024-03-29T04:45:02.432 INFO:journalctl@ceph.mon.a.smithi181.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:45:02.432 INFO:journalctl@ceph.mon.a.smithi181.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:45:02.432 INFO:journalctl@ceph.mon.a.smithi181.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 149.7 0.00 0.00 1 0.001 0 0 2024-03-29T04:45:02.432 INFO:journalctl@ceph.mon.a.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:45:02.432 INFO:journalctl@ceph.mon.a.smithi181.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-29T04:45:02.432 INFO:journalctl@ceph.mon.a.smithi181.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:45:02.432 INFO:journalctl@ceph.mon.a.smithi181.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:45:02.432 INFO:journalctl@ceph.mon.a.smithi181.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:45:02.432 INFO:journalctl@ceph.mon.a.smithi181.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:45:02.432 INFO:journalctl@ceph.mon.a.smithi181.stdout: Cumulative compaction: 0.00 GB write, 26.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:45:02.432 INFO:journalctl@ceph.mon.a.smithi181.stdout: Interval compaction: 0.00 GB write, 26.13 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:45:02.433 INFO:journalctl@ceph.mon.a.smithi181.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:45:02.433 INFO:journalctl@ceph.mon.a.smithi181.stdout: 2024-03-29T04:45:02.433 INFO:journalctl@ceph.mon.a.smithi181.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:45:02.433 INFO:journalctl@ceph.mon.a.smithi181.stdout: 2024-03-29T04:45:02.433 INFO:journalctl@ceph.mon.a.smithi181.stdout: ** Compaction Stats [default] ** 2024-03-29T04:45:02.433 INFO:journalctl@ceph.mon.a.smithi181.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:45:02.433 INFO:journalctl@ceph.mon.a.smithi181.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:45:02.433 INFO:journalctl@ceph.mon.a.smithi181.stdout: L0 2/0 80.69 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.7 0.00 0.00 1 0.001 0 0 2024-03-29T04:45:02.433 INFO:journalctl@ceph.mon.a.smithi181.stdout: Sum 2/0 80.69 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.7 0.00 0.00 1 0.001 0 0 2024-03-29T04:45:02.433 INFO:journalctl@ceph.mon.a.smithi181.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-03-29T04:45:02.433 INFO:journalctl@ceph.mon.a.smithi181.stdout: 2024-03-29T04:45:02.433 INFO:journalctl@ceph.mon.a.smithi181.stdout: ** Compaction Stats [default] ** 2024-03-29T04:45:02.433 INFO:journalctl@ceph.mon.a.smithi181.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:45:02.433 INFO:journalctl@ceph.mon.a.smithi181.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:45:02.434 INFO:journalctl@ceph.mon.a.smithi181.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 149.7 0.00 0.00 1 0.001 0 0 2024-03-29T04:45:02.434 INFO:journalctl@ceph.mon.a.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:45:02.434 INFO:journalctl@ceph.mon.a.smithi181.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-29T04:45:02.434 INFO:journalctl@ceph.mon.a.smithi181.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:45:02.434 INFO:journalctl@ceph.mon.a.smithi181.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:45:02.434 INFO:journalctl@ceph.mon.a.smithi181.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:45:02.434 INFO:journalctl@ceph.mon.a.smithi181.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:45:02.434 INFO:journalctl@ceph.mon.a.smithi181.stdout: Cumulative compaction: 0.00 GB write, 25.78 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:45:02.434 INFO:journalctl@ceph.mon.a.smithi181.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:45:02.434 INFO:journalctl@ceph.mon.a.smithi181.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:45:02.434 INFO:journalctl@ceph.mon.a.smithi181.stdout: 2024-03-29T04:45:02.434 INFO:journalctl@ceph.mon.a.smithi181.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:45:02.434 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: starting mon.a rank 0 at public addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] at bind addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:45:02.434 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: mon.a@-1(???) e1 preinit fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:45:02.434 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: mon.a@-1(???).mds e1 new map 2024-03-29T04:45:02.434 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: mon.a@-1(???).mds e1 print_map 2024-03-29T04:45:02.435 INFO:journalctl@ceph.mon.a.smithi181.stdout: e1 2024-03-29T04:45:02.435 INFO:journalctl@ceph.mon.a.smithi181.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-29T04:45:02.435 INFO:journalctl@ceph.mon.a.smithi181.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:45:02.435 INFO:journalctl@ceph.mon.a.smithi181.stdout: legacy client fscid: -1 2024-03-29T04:45:02.435 INFO:journalctl@ceph.mon.a.smithi181.stdout: 2024-03-29T04:45:02.435 INFO:journalctl@ceph.mon.a.smithi181.stdout: No filesystems configured 2024-03-29T04:45:02.435 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-29T04:45:02.435 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:45:02.435 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:45:02.435 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:45:02.435 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-03-29T04:45:02.435 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 bash[29727]: 3e3936015cad924909bce492f26f753eb72f818dfd57ce0d65915978f085b479 2024-03-29T04:45:02.435 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 systemd[1]: Started Ceph mon.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:45:02.436 INFO:teuthology.orchestra.run.smithi181.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-03-29T04:45:02.695 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: mon.a is new leader, mons a in quorum (ranks 0) 2024-03-29T04:45:02.695 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: monmap e1: 1 mons at {a=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:45:02.695 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: fsmap 2024-03-29T04:45:02.695 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: osdmap e1: 0 total, 0 up, 0 in 2024-03-29T04:45:02.695 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:02 smithi181 ceph-mon[29765]: mgrmap e1: no daemons active 2024-03-29T04:45:03.537 INFO:teuthology.orchestra.run.smithi181.stdout:Wrote config to /etc/ceph/ceph.conf 2024-03-29T04:45:03.537 INFO:teuthology.orchestra.run.smithi181.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:45:03.537 INFO:teuthology.orchestra.run.smithi181.stdout:Creating mgr... 2024-03-29T04:45:03.538 INFO:teuthology.orchestra.run.smithi181.stdout:Verifying port 9283 ... 2024-03-29T04:45:03.712 INFO:teuthology.orchestra.run.smithi181.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.y 2024-03-29T04:45:03.712 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stderr Failed to reset failed state of unit ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.y.service: Unit ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.y.service not loaded. 2024-03-29T04:45:03.854 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-dfcecab8-ed86-11ee-95d0-87774f69a715.target.wants/ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.y.service → /etc/systemd/system/ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@.service. 2024-03-29T04:45:04.195 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:03 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1943949198' entity='client.admin' 2024-03-29T04:45:04.195 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:04 smithi181 podman[30004]: 2024-03-29 04:45:03.995041608 +0000 UTC m=+0.017500843 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph:quincy 2024-03-29T04:45:05.695 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:05 smithi181 podman[30004]: 2024-03-29 04:45:05.216093064 +0000 UTC m=+1.238552287 container create 37fa586e0972b1f4c4f2cd944e171c82fcaea660ae0d6ad53edca245ed94830d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-03-29T04:45:06.042 INFO:teuthology.orchestra.run.smithi181.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-03-29T04:45:06.042 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stdout disabled 2024-03-29T04:45:06.049 INFO:teuthology.orchestra.run.smithi181.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-03-29T04:45:06.049 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stdout inactive 2024-03-29T04:45:06.049 INFO:teuthology.orchestra.run.smithi181.stdout:firewalld.service is not enabled 2024-03-29T04:45:06.049 INFO:teuthology.orchestra.run.smithi181.stdout:Not possible to enable service . firewalld.service is not available 2024-03-29T04:45:06.057 INFO:teuthology.orchestra.run.smithi181.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-03-29T04:45:06.057 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stdout disabled 2024-03-29T04:45:06.065 INFO:teuthology.orchestra.run.smithi181.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-03-29T04:45:06.065 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stdout inactive 2024-03-29T04:45:06.065 INFO:teuthology.orchestra.run.smithi181.stdout:firewalld.service is not enabled 2024-03-29T04:45:06.065 INFO:teuthology.orchestra.run.smithi181.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2024-03-29T04:45:06.066 INFO:teuthology.orchestra.run.smithi181.stdout:Waiting for mgr to start... 2024-03-29T04:45:06.066 INFO:teuthology.orchestra.run.smithi181.stdout:Waiting for mgr... 2024-03-29T04:45:06.190 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:05 smithi181 podman[30004]: 2024-03-29 04:45:05.922033892 +0000 UTC m=+1.944493120 container init 37fa586e0972b1f4c4f2cd944e171c82fcaea660ae0d6ad53edca245ed94830d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-03-29T04:45:06.190 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:05 smithi181 podman[30004]: 2024-03-29 04:45:05.928344736 +0000 UTC m=+1.950803961 container start 37fa586e0972b1f4c4f2cd944e171c82fcaea660ae0d6ad53edca245ed94830d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240326, RELEASE=quincy-4213e17, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:45:06.190 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:06 smithi181 bash[30004]: 37fa586e0972b1f4c4f2cd944e171c82fcaea660ae0d6ad53edca245ed94830d 2024-03-29T04:45:06.191 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:06 smithi181 systemd[1]: Started Ceph mgr.y for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:45:06.945 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:06 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/503273894' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-29T04:45:06.945 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:06 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:06.454+0000 7fc7746661c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:45:06.945 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:06 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:06.548+0000 7fc7746661c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:45:07.227 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:07.197+0000 7fc7746661c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:45:07.232 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 2024-03-29T04:45:07.232 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:45:07.232 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "fsid": "dfcecab8-ed86-11ee-95d0-87774f69a715", 2024-03-29T04:45:07.232 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "health": { 2024-03-29T04:45:07.232 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-29T04:45:07.232 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-29T04:45:07.232 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-29T04:45:07.232 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:07.232 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-29T04:45:07.232 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-29T04:45:07.232 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 0 2024-03-29T04:45:07.232 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:45:07.232 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-29T04:45:07.233 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "a" 2024-03-29T04:45:07.233 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:45:07.233 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "quorum_age": 4, 2024-03-29T04:45:07.233 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-29T04:45:07.233 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:45:07.233 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-03-29T04:45:07.233 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-29T04:45:07.233 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:07.233 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-29T04:45:07.233 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:45:07.233 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-29T04:45:07.233 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-29T04:45:07.233 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-29T04:45:07.233 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-29T04:45:07.233 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-29T04:45:07.233 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-29T04:45:07.237 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:07.237 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-29T04:45:07.237 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-29T04:45:07.237 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-29T04:45:07.237 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-29T04:45:07.237 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-29T04:45:07.237 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-29T04:45:07.237 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-29T04:45:07.237 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-29T04:45:07.237 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-29T04:45:07.237 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:07.237 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-29T04:45:07.237 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:45:07.237 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-29T04:45:07.237 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-29T04:45:07.238 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:07.238 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-29T04:45:07.238 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "available": false, 2024-03-29T04:45:07.238 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-29T04:45:07.238 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-29T04:45:07.238 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-29T04:45:07.238 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-29T04:45:07.238 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "restful" 2024-03-29T04:45:07.238 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:45:07.238 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:45:07.238 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:07.238 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-29T04:45:07.238 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:45:07.238 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "modified": "2024-03-29T04:44:49.014794+0000", 2024-03-29T04:45:07.238 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:45:07.239 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:07.239 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-29T04:45:07.239 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:45:07.239 INFO:teuthology.orchestra.run.smithi181.stdout:mgr not available, waiting (1/15)... 2024-03-29T04:45:07.517 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:07.288+0000 7fc7746661c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:45:07.517 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:07.370+0000 7fc7746661c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:45:07.945 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:07.517+0000 7fc7746661c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:45:07.945 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:07.597+0000 7fc7746661c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:45:07.945 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:07.763+0000 7fc7746661c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:45:08.208 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:07.954+0000 7fc7746661c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:45:08.208 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:08.036+0000 7fc7746661c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:45:08.208 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:08.126+0000 7fc7746661c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:45:08.208 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:08.208+0000 7fc7746661c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:45:09.487 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:09 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:09.247+0000 7fc7746661c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:45:09.488 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:09 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:09.384+0000 7fc7746661c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:45:09.755 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:09 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:09.586+0000 7fc7746661c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:45:10.011 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:09 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/105875407' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-29T04:45:10.011 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:09 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:09.827+0000 7fc7746661c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:45:10.445 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:10 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:10.012+0000 7fc7746661c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:45:10.445 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:10 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:10.193+0000 7fc7746661c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:45:10.469 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 2024-03-29T04:45:10.471 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:45:10.472 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "fsid": "dfcecab8-ed86-11ee-95d0-87774f69a715", 2024-03-29T04:45:10.472 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "health": { 2024-03-29T04:45:10.472 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-29T04:45:10.472 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-29T04:45:10.472 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-29T04:45:10.472 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:10.472 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-29T04:45:10.472 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-29T04:45:10.472 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 0 2024-03-29T04:45:10.472 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:45:10.472 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-29T04:45:10.472 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "a" 2024-03-29T04:45:10.472 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:45:10.472 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "quorum_age": 7, 2024-03-29T04:45:10.473 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-29T04:45:10.473 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:45:10.473 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-03-29T04:45:10.473 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-29T04:45:10.473 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:10.473 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-29T04:45:10.473 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:45:10.473 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-29T04:45:10.473 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-29T04:45:10.473 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-29T04:45:10.473 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-29T04:45:10.473 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-29T04:45:10.473 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-29T04:45:10.477 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:10.477 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-29T04:45:10.477 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-29T04:45:10.477 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-29T04:45:10.477 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-29T04:45:10.477 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-29T04:45:10.477 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-29T04:45:10.477 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-29T04:45:10.477 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-29T04:45:10.477 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-29T04:45:10.478 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:10.478 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-29T04:45:10.478 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:45:10.478 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-29T04:45:10.478 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-29T04:45:10.478 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:10.478 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-29T04:45:10.478 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "available": false, 2024-03-29T04:45:10.478 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-29T04:45:10.478 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-29T04:45:10.478 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-29T04:45:10.478 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-29T04:45:10.479 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "restful" 2024-03-29T04:45:10.479 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:45:10.479 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:45:10.479 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:10.479 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-29T04:45:10.479 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:45:10.479 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "modified": "2024-03-29T04:44:49.014794+0000", 2024-03-29T04:45:10.479 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:45:10.479 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:10.479 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-29T04:45:10.479 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:45:10.479 INFO:teuthology.orchestra.run.smithi181.stdout:mgr not available, waiting (2/15)... 2024-03-29T04:45:10.945 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:10 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:10.454+0000 7fc7746661c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:45:10.945 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:10 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:10.542+0000 7fc7746661c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:45:11.445 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:10 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:10.945+0000 7fc7746661c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:45:11.993 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:11.710+0000 7fc7746661c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:45:11.993 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:11.993+0000 7fc7746661c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:45:12.599 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:12 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:12.306+0000 7fc7746661c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:45:12.599 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:12 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:12.497+0000 7fc7746661c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:45:12.599 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:12 smithi181 ceph-mon[29765]: Activating manager daemon y 2024-03-29T04:45:12.599 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:12 smithi181 ceph-mon[29765]: mgrmap e2: y(active, starting, since 0.00298521s) 2024-03-29T04:45:12.599 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:12 smithi181 ceph-mon[29765]: from='mgr.14100 172.21.15.181:0/347809593' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:45:12.599 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:12 smithi181 ceph-mon[29765]: from='mgr.14100 172.21.15.181:0/347809593' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:45:12.599 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:12 smithi181 ceph-mon[29765]: from='mgr.14100 172.21.15.181:0/347809593' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:45:12.599 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:12 smithi181 ceph-mon[29765]: from='mgr.14100 172.21.15.181:0/347809593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:45:12.599 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:12 smithi181 ceph-mon[29765]: from='mgr.14100 172.21.15.181:0/347809593' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:45:12.600 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:12 smithi181 ceph-mon[29765]: Manager daemon y is now available 2024-03-29T04:45:12.600 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:12 smithi181 ceph-mon[29765]: from='mgr.14100 172.21.15.181:0/347809593' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:45:12.600 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:12 smithi181 ceph-mon[29765]: from='mgr.14100 172.21.15.181:0/347809593' entity='mgr.y' 2024-03-29T04:45:12.600 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:12 smithi181 ceph-mon[29765]: from='mgr.14100 172.21.15.181:0/347809593' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:45:12.600 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:12 smithi181 ceph-mon[29765]: from='mgr.14100 172.21.15.181:0/347809593' entity='mgr.y' 2024-03-29T04:45:12.600 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:12 smithi181 ceph-mon[29765]: from='mgr.14100 172.21.15.181:0/347809593' entity='mgr.y' 2024-03-29T04:45:13.598 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:13 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/575346804' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-29T04:45:13.599 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:13 smithi181 ceph-mon[29765]: mgrmap e3: y(active, since 1.00604s) 2024-03-29T04:45:13.604 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 2024-03-29T04:45:13.604 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:45:13.604 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "fsid": "dfcecab8-ed86-11ee-95d0-87774f69a715", 2024-03-29T04:45:13.604 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "health": { 2024-03-29T04:45:13.604 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-29T04:45:13.604 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-29T04:45:13.604 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-29T04:45:13.604 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:13.605 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-29T04:45:13.605 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-29T04:45:13.605 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 0 2024-03-29T04:45:13.605 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:45:13.605 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-29T04:45:13.606 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "a" 2024-03-29T04:45:13.606 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:45:13.606 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "quorum_age": 10, 2024-03-29T04:45:13.606 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-29T04:45:13.606 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:45:13.606 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-03-29T04:45:13.606 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-29T04:45:13.606 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:13.606 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-29T04:45:13.606 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:45:13.606 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-29T04:45:13.607 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-29T04:45:13.607 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-29T04:45:13.607 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-29T04:45:13.607 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-29T04:45:13.607 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-29T04:45:13.611 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:13.611 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-29T04:45:13.611 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-29T04:45:13.611 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-29T04:45:13.611 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-29T04:45:13.611 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-29T04:45:13.611 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-29T04:45:13.611 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-29T04:45:13.611 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-29T04:45:13.611 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-29T04:45:13.611 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:13.611 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-29T04:45:13.612 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:45:13.612 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-29T04:45:13.612 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-29T04:45:13.612 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:13.612 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-29T04:45:13.612 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "available": false, 2024-03-29T04:45:13.612 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-29T04:45:13.612 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-29T04:45:13.612 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-29T04:45:13.612 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-29T04:45:13.612 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "restful" 2024-03-29T04:45:13.612 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:45:13.612 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:45:13.612 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:13.612 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-29T04:45:13.612 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:45:13.613 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "modified": "2024-03-29T04:44:49.014794+0000", 2024-03-29T04:45:13.613 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:45:13.613 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:13.613 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-29T04:45:13.613 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:45:13.613 INFO:teuthology.orchestra.run.smithi181.stdout:mgr not available, waiting (3/15)... 2024-03-29T04:45:15.876 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:15 smithi181 ceph-mon[29765]: mgrmap e4: y(active, since 2s) 2024-03-29T04:45:16.670 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:16 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2222265833' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-29T04:45:16.960 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 2024-03-29T04:45:16.960 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:45:16.960 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "fsid": "dfcecab8-ed86-11ee-95d0-87774f69a715", 2024-03-29T04:45:16.960 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "health": { 2024-03-29T04:45:16.960 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-29T04:45:16.960 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-29T04:45:16.960 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-29T04:45:16.960 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:16.960 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-29T04:45:16.960 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-29T04:45:16.960 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 0 2024-03-29T04:45:16.960 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:45:16.961 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-29T04:45:16.961 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "a" 2024-03-29T04:45:16.961 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:45:16.961 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "quorum_age": 14, 2024-03-29T04:45:16.961 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-29T04:45:16.961 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:45:16.961 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-03-29T04:45:16.961 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-29T04:45:16.961 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:16.961 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-29T04:45:16.961 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:45:16.962 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-29T04:45:16.962 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-29T04:45:16.962 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-29T04:45:16.962 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-29T04:45:16.962 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-29T04:45:16.962 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-29T04:45:16.965 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:16.965 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-29T04:45:16.965 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-29T04:45:16.965 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-29T04:45:16.965 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-29T04:45:16.965 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-29T04:45:16.966 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-29T04:45:16.966 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-29T04:45:16.966 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-29T04:45:16.966 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-29T04:45:16.966 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:16.966 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-29T04:45:16.966 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:45:16.966 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-29T04:45:16.966 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-29T04:45:16.966 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:16.966 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-29T04:45:16.966 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-29T04:45:16.966 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-29T04:45:16.967 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-29T04:45:16.967 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-29T04:45:16.967 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-29T04:45:16.967 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "restful" 2024-03-29T04:45:16.967 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:45:16.967 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:45:16.967 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:16.967 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-29T04:45:16.967 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:45:16.967 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "modified": "2024-03-29T04:44:49.014794+0000", 2024-03-29T04:45:16.967 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:45:16.967 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:45:16.967 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-29T04:45:16.967 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:45:16.967 INFO:teuthology.orchestra.run.smithi181.stdout:mgr is available 2024-03-29T04:45:17.852 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:17 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3632616445' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-29T04:45:18.132 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 2024-03-29T04:45:18.132 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout [global] 2024-03-29T04:45:18.132 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout fsid = dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:45:18.132 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-03-29T04:45:18.133 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-03-29T04:45:18.133 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-03-29T04:45:18.133 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-03-29T04:45:18.133 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 2024-03-29T04:45:18.133 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout [mgr] 2024-03-29T04:45:18.133 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-03-29T04:45:18.133 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 2024-03-29T04:45:18.133 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout [osd] 2024-03-29T04:45:18.133 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-03-29T04:45:18.133 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-03-29T04:45:18.133 INFO:teuthology.orchestra.run.smithi181.stdout:Enabling cephadm module... 2024-03-29T04:45:19.176 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:18 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2518703601' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-03-29T04:45:19.176 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ignoring --setuser ceph since I am not root 2024-03-29T04:45:19.176 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ignoring --setgroup ceph since I am not root 2024-03-29T04:45:19.445 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:19 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:19.400+0000 7f621c9291c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:45:19.754 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:19 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:19.502+0000 7f621c9291c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:45:20.108 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:19 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2518703601' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-03-29T04:45:20.108 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:19 smithi181 ceph-mon[29765]: mgrmap e5: y(active, since 6s) 2024-03-29T04:45:20.445 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:20.152+0000 7f621c9291c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:45:20.445 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:20.239+0000 7f621c9291c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:45:20.445 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:20.320+0000 7f621c9291c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:45:20.725 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:20.472+0000 7f621c9291c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:45:20.725 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:20.559+0000 7f621c9291c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:45:20.725 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:20.726+0000 7f621c9291c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:45:20.736 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:45:20.737 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-03-29T04:45:20.737 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-29T04:45:20.737 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-03-29T04:45:20.737 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-03-29T04:45:20.737 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:45:20.737 INFO:teuthology.orchestra.run.smithi181.stdout:Waiting for the mgr to restart... 2024-03-29T04:45:20.737 INFO:teuthology.orchestra.run.smithi181.stdout:Waiting for mgr epoch 5... 2024-03-29T04:45:20.975 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:20 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2144867265' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-29T04:45:20.976 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:20.914+0000 7f621c9291c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:45:21.445 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:21.003+0000 7f621c9291c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:45:21.445 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:21.097+0000 7f621c9291c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:45:21.445 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:21.186+0000 7f621c9291c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:45:22.537 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:22 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:22.219+0000 7f621c9291c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:45:22.537 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:22 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:22.346+0000 7f621c9291c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:45:22.537 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:22 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:22.537+0000 7f621c9291c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:45:22.941 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:22 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:22.766+0000 7f621c9291c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:45:22.941 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:22 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:22.941+0000 7f621c9291c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:45:23.195 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:23 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:23.120+0000 7f621c9291c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:45:23.695 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:23 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:23.379+0000 7f621c9291c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:45:23.695 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:23 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:23.464+0000 7f621c9291c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:45:24.195 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:23 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:23.876+0000 7f621c9291c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:45:24.928 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:24 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:24.640+0000 7f621c9291c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:45:24.928 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:24 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:24.928+0000 7f621c9291c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:45:25.695 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:25.275+0000 7f621c9291c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:45:25.695 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:25.465+0000 7f621c9291c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:45:25.695 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: Active manager daemon y restarted 2024-03-29T04:45:25.695 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: Activating manager daemon y 2024-03-29T04:45:25.695 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: osdmap e2: 0 total, 0 up, 0 in 2024-03-29T04:45:25.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: mgrmap e6: y(active, starting, since 0.0032454s) 2024-03-29T04:45:25.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:45:25.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:45:25.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:45:25.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:45:25.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:45:25.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: Manager daemon y is now available 2024-03-29T04:45:25.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' 2024-03-29T04:45:25.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' 2024-03-29T04:45:25.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:25.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:25.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:25.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:45:25.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:25 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:45:26.780 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:26 smithi181 ceph-mon[29765]: Found migration_current of "None". Setting to last migration. 2024-03-29T04:45:26.780 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:26 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' 2024-03-29T04:45:26.780 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:26 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:26.781 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:26 smithi181 ceph-mon[29765]: mgrmap e7: y(active, since 1.00614s) 2024-03-29T04:45:27.018 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:45:27.018 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-03-29T04:45:27.018 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "initialized": true 2024-03-29T04:45:27.018 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:45:27.018 INFO:teuthology.orchestra.run.smithi181.stdout:mgr epoch 5 is available 2024-03-29T04:45:27.018 INFO:teuthology.orchestra.run.smithi181.stdout:Setting orchestrator backend to cephadm... 2024-03-29T04:45:27.945 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:27 smithi181 ceph-mon[29765]: [29/Mar/2024:04:45:25] ENGINE Bus STARTING 2024-03-29T04:45:27.945 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:27 smithi181 ceph-mon[29765]: [29/Mar/2024:04:45:26] ENGINE Serving on https://172.21.15.181:7150 2024-03-29T04:45:27.945 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:27 smithi181 ceph-mon[29765]: [29/Mar/2024:04:45:26] ENGINE Bus STARTED 2024-03-29T04:45:27.945 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:27 smithi181 ceph-mon[29765]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-29T04:45:27.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:27 smithi181 ceph-mon[29765]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-29T04:45:28.910 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:28 smithi181 ceph-mon[29765]: mgrmap e8: y(active, since 2s) 2024-03-29T04:45:28.910 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:28 smithi181 ceph-mon[29765]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:45:28.910 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:28 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' 2024-03-29T04:45:28.911 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:28 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:29.431 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout value unchanged 2024-03-29T04:45:29.432 INFO:teuthology.orchestra.run.smithi181.stdout:Generating ssh key... 2024-03-29T04:45:29.684 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:29 smithi181 ceph-mon[29765]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:45:30.493 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: Generating public/private rsa key pair. 2024-03-29T04:45:30.493 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: Your identification has been saved in /tmp/tmpcr6c2ith/key. 2024-03-29T04:45:30.493 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: Your public key has been saved in /tmp/tmpcr6c2ith/key.pub. 2024-03-29T04:45:30.493 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: The key fingerprint is: 2024-03-29T04:45:30.493 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: SHA256:YcK2+vBCqevY3y4BdUIK3B+/wXCwy/SjNDOzo8vsmcw ceph-dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:45:30.493 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: The key's randomart image is: 2024-03-29T04:45:30.493 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: +---[RSA 3072]----+ 2024-03-29T04:45:30.494 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: |o ..... | 2024-03-29T04:45:30.494 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: | o oo+o. | 2024-03-29T04:45:30.494 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: | ...=X o | 2024-03-29T04:45:30.494 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: | . oooB . | 2024-03-29T04:45:30.494 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: | . X.oS | 2024-03-29T04:45:30.494 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: | =.B.. | 2024-03-29T04:45:30.494 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: | oo= | 2024-03-29T04:45:30.494 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: | o* *=. | 2024-03-29T04:45:30.494 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: |..=Eo+= | 2024-03-29T04:45:30.494 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: +----[SHA256]-----+ 2024-03-29T04:45:31.445 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:31 smithi181 ceph-mon[29765]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:45:31.445 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:31 smithi181 ceph-mon[29765]: Generating ssh key... 2024-03-29T04:45:31.445 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:31 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' 2024-03-29T04:45:31.445 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:31 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' 2024-03-29T04:45:31.960 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDkAcLiN5OL9FkxpFWSlDOwn3Q1VrmM/hLaKxLcXMMjyacEdsp/rIucDLq3/D5s4lB8riTXBQkVmlVkC82dAjIy/tkHcpbc4NEzsSt8M1G6iU8Jz/eGrBF+uqnILR3y0VkNHjafaYhxn/6awRArbVYBqOuvKw6TeAQVGModuk6rPldZjjrraRhU4F5Genf8SN79eCc+bwHYnbvV1J6HS/oJb8yacf06P1IpS6IY00m6y9YGiYHseRL2870pIahZIW/Nx1SV9W6I7IqxTNs2q8advdpTGfjsJndcVaaPsKM4yVo/zkwzeW+Sk5IZkOsZN8HmpvX1P+YiyxvlvK7gBigxO74WhfVC/CBn9H57n08ftGBPwOrYtLAytdUnKRaZE5RwacUUrj3Cqopdw5dBp22RDcFMh9i1IiiiRx172eirpHCh9HsveJFxdNvtLM1UibkHHLhIogoq26d5UkA9gDXr9CM/ySdD5x3n+ylum8Xl/Rs6F4ITtIxxxy9JO6InZq8= ceph-dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:45:31.960 INFO:teuthology.orchestra.run.smithi181.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-03-29T04:45:31.960 INFO:teuthology.orchestra.run.smithi181.stdout:Adding key to root@localhost authorized_keys... 2024-03-29T04:45:31.961 INFO:teuthology.orchestra.run.smithi181.stdout:Adding host smithi181... 2024-03-29T04:45:32.319 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:32 smithi181 ceph-mon[29765]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:45:33.466 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:33 smithi181 ceph-mon[29765]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi181", "addr": "172.21.15.181", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:45:34.445 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:34 smithi181 ceph-mon[29765]: Deploying cephadm binary to smithi181 2024-03-29T04:45:35.185 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout Added host 'smithi181' with addr '172.21.15.181' 2024-03-29T04:45:35.185 INFO:teuthology.orchestra.run.smithi181.stdout:Deploying unmanaged mon service... 2024-03-29T04:45:35.852 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:35 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' 2024-03-29T04:45:35.853 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:35 smithi181 ceph-mon[29765]: Added host smithi181 2024-03-29T04:45:35.853 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:35 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:36.411 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-03-29T04:45:36.411 INFO:teuthology.orchestra.run.smithi181.stdout:Deploying unmanaged mgr service... 2024-03-29T04:45:37.157 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:36 smithi181 ceph-mon[29765]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:45:37.157 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:36 smithi181 ceph-mon[29765]: Saving service mon spec with placement count:5 2024-03-29T04:45:37.157 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:36 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' 2024-03-29T04:45:38.495 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-03-29T04:45:38.731 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:38 smithi181 ceph-mon[29765]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:45:38.731 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:38 smithi181 ceph-mon[29765]: Saving service mgr spec with placement count:2 2024-03-29T04:45:38.731 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:38 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' 2024-03-29T04:45:38.731 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:38 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' 2024-03-29T04:45:38.731 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:38 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' 2024-03-29T04:45:40.423 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:40 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1026826523' entity='client.admin' 2024-03-29T04:45:41.192 INFO:teuthology.orchestra.run.smithi181.stdout:Enabling the dashboard module... 2024-03-29T04:45:41.945 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:41 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2299561987' entity='client.admin' 2024-03-29T04:45:41.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:41 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' 2024-03-29T04:45:42.945 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:42 smithi181 ceph-mon[29765]: from='mgr.14120 172.21.15.181:0/4221127239' entity='mgr.y' 2024-03-29T04:45:42.945 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:42 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3987121891' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-03-29T04:45:42.946 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:42 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ignoring --setuser ceph since I am not root 2024-03-29T04:45:42.946 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:42 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ignoring --setgroup ceph since I am not root 2024-03-29T04:45:43.314 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:43 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:43.217+0000 7f0c9d09a1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:45:43.314 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:43 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:43.314+0000 7f0c9d09a1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:45:43.823 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:43 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3987121891' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-03-29T04:45:43.824 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:43 smithi181 ceph-mon[29765]: mgrmap e9: y(active, since 17s) 2024-03-29T04:45:44.289 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:43 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:43.957+0000 7f0c9d09a1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:45:44.289 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:44 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:44.049+0000 7f0c9d09a1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:45:44.289 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:44 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:44.135+0000 7f0c9d09a1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:45:44.289 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:44 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:44.288+0000 7f0c9d09a1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:45:44.543 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:44 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:44.372+0000 7f0c9d09a1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:45:44.543 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:44 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:44.543+0000 7f0c9d09a1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:45:44.810 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:44 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1804102410' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-29T04:45:44.811 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:44 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:44.726+0000 7f0c9d09a1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:45:44.811 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:44 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:44.810+0000 7f0c9d09a1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:45:44.854 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:45:44.854 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-03-29T04:45:44.854 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-29T04:45:44.854 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-03-29T04:45:44.854 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-03-29T04:45:44.854 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:45:44.854 INFO:teuthology.orchestra.run.smithi181.stdout:Waiting for the mgr to restart... 2024-03-29T04:45:44.855 INFO:teuthology.orchestra.run.smithi181.stdout:Waiting for mgr epoch 9... 2024-03-29T04:45:45.102 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:44 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:44.907+0000 7f0c9d09a1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:45:45.102 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:44 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:44.992+0000 7f0c9d09a1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:45:46.353 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:46 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:46.037+0000 7f0c9d09a1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:45:46.353 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:46 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:46.164+0000 7f0c9d09a1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:45:46.353 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:46 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:46.352+0000 7f0c9d09a1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:45:46.695 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:46 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:46.581+0000 7f0c9d09a1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:45:47.187 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:46 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:46.754+0000 7f0c9d09a1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:45:47.187 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:46 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:46.931+0000 7f0c9d09a1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:45:47.445 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:47 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:47.187+0000 7f0c9d09a1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:45:47.445 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:47 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:47.271+0000 7f0c9d09a1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:45:47.945 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:47 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:47.661+0000 7f0c9d09a1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:45:48.695 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:48.429+0000 7f0c9d09a1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:45:49.038 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:48.721+0000 7f0c9d09a1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:45:49.292 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:49 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:49.038+0000 7f0c9d09a1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:45:49.292 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:45:49 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:45:49.232+0000 7f0c9d09a1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:45:49.292 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:49 smithi181 ceph-mon[29765]: Active manager daemon y restarted 2024-03-29T04:45:49.292 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:49 smithi181 ceph-mon[29765]: Activating manager daemon y 2024-03-29T04:45:49.292 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:49 smithi181 ceph-mon[29765]: osdmap e3: 0 total, 0 up, 0 in 2024-03-29T04:45:49.292 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:49 smithi181 ceph-mon[29765]: mgrmap e10: y(active, starting, since 0.00548108s) 2024-03-29T04:45:49.292 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:49 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:45:49.292 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:49 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:45:49.292 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:49 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:45:49.292 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:49 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:45:49.292 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:49 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:45:49.292 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:49 smithi181 ceph-mon[29765]: Manager daemon y is now available 2024-03-29T04:45:49.293 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:49 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:49.293 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:49 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:49.293 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:49 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:45:49.293 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:49 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:45:50.902 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:50 smithi181 ceph-mon[29765]: [29/Mar/2024:04:45:49] ENGINE Bus STARTING 2024-03-29T04:45:50.902 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:50 smithi181 ceph-mon[29765]: [29/Mar/2024:04:45:49] ENGINE Serving on https://172.21.15.181:7150 2024-03-29T04:45:50.902 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:50 smithi181 ceph-mon[29765]: [29/Mar/2024:04:45:49] ENGINE Bus STARTED 2024-03-29T04:45:50.902 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:50 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:45:50.902 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:50 smithi181 ceph-mon[29765]: mgrmap e11: y(active, since 1.00545s) 2024-03-29T04:45:50.904 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:45:50.904 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-03-29T04:45:50.904 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout "initialized": true 2024-03-29T04:45:50.904 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:45:50.904 INFO:teuthology.orchestra.run.smithi181.stdout:mgr epoch 9 is available 2024-03-29T04:45:50.904 INFO:teuthology.orchestra.run.smithi181.stdout:Generating a dashboard self-signed certificate... 2024-03-29T04:45:51.764 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:51 smithi181 ceph-mon[29765]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-29T04:45:51.764 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:51 smithi181 ceph-mon[29765]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-29T04:45:51.764 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:51 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:45:52.277 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-03-29T04:45:52.277 INFO:teuthology.orchestra.run.smithi181.stdout:Creating initial admin user... 2024-03-29T04:45:52.945 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:52 smithi181 ceph-mon[29765]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:45:52.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:52 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:45:52.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:52 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:45:52.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:52 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:45:52.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:52 smithi181 ceph-mon[29765]: mgrmap e12: y(active, since 2s) 2024-03-29T04:45:54.293 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$2PBlxMRfpWHMmK0jA3cVPeozPiOZvObtCkbI.dY0kAn/WVaWrZGvu", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1711687553, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-03-29T04:45:54.294 INFO:teuthology.orchestra.run.smithi181.stdout:Fetching dashboard port number... 2024-03-29T04:45:54.418 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:54 smithi181 ceph-mon[29765]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:45:54.418 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:54 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:45:55.270 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:55 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/128461727' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-03-29T04:45:55.524 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stdout 8443 2024-03-29T04:45:55.533 INFO:teuthology.orchestra.run.smithi181.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-03-29T04:45:55.533 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stdout disabled 2024-03-29T04:45:55.540 INFO:teuthology.orchestra.run.smithi181.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-03-29T04:45:55.540 INFO:teuthology.orchestra.run.smithi181.stdout:systemctl: stdout inactive 2024-03-29T04:45:55.541 INFO:teuthology.orchestra.run.smithi181.stdout:firewalld.service is not enabled 2024-03-29T04:45:55.541 INFO:teuthology.orchestra.run.smithi181.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-03-29T04:45:55.542 INFO:teuthology.orchestra.run.smithi181.stdout:Ceph Dashboard is now available at: 2024-03-29T04:45:55.542 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:45:55.542 INFO:teuthology.orchestra.run.smithi181.stdout: URL: https://smithi181.front.sepia.ceph.com:8443/ 2024-03-29T04:45:55.543 INFO:teuthology.orchestra.run.smithi181.stdout: User: admin 2024-03-29T04:45:55.543 INFO:teuthology.orchestra.run.smithi181.stdout: Password: ltezud191a 2024-03-29T04:45:55.543 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:45:55.543 INFO:teuthology.orchestra.run.smithi181.stdout:Saving cluster configuration to /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config directory 2024-03-29T04:45:55.543 INFO:teuthology.orchestra.run.smithi181.stdout:Enabling autotune for osd_memory_target 2024-03-29T04:45:57.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:57 smithi181 ceph-mon[29765]: mgrmap e13: y(active, since 6s) 2024-03-29T04:45:57.970 INFO:teuthology.orchestra.run.smithi181.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-03-29T04:45:57.970 INFO:teuthology.orchestra.run.smithi181.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-03-29T04:45:57.970 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:45:57.970 INFO:teuthology.orchestra.run.smithi181.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:45:57.970 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:45:57.970 INFO:teuthology.orchestra.run.smithi181.stdout:Or, if you are only running a single cluster on this host: 2024-03-29T04:45:57.970 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:45:57.971 INFO:teuthology.orchestra.run.smithi181.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-03-29T04:45:57.971 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:45:57.971 INFO:teuthology.orchestra.run.smithi181.stdout:Please consider enabling telemetry to help improve Ceph: 2024-03-29T04:45:57.971 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:45:57.971 INFO:teuthology.orchestra.run.smithi181.stdout: ceph telemetry on 2024-03-29T04:45:57.971 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:45:57.971 INFO:teuthology.orchestra.run.smithi181.stdout:For more information see: 2024-03-29T04:45:57.971 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:45:57.971 INFO:teuthology.orchestra.run.smithi181.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-03-29T04:45:57.971 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:45:57.971 INFO:teuthology.orchestra.run.smithi181.stdout:Bootstrap complete. 2024-03-29T04:45:58.010 INFO:tasks.cephadm:Fetching config... 2024-03-29T04:45:58.010 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-03-29T04:45:58.010 DEBUG:teuthology.orchestra.run.smithi181:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-03-29T04:45:58.027 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-03-29T04:45:58.027 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-03-29T04:45:58.027 DEBUG:teuthology.orchestra.run.smithi181:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-03-29T04:45:58.081 INFO:tasks.cephadm:Fetching mon keyring... 2024-03-29T04:45:58.082 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-03-29T04:45:58.082 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/keyring of=/dev/stdout 2024-03-29T04:45:58.149 INFO:tasks.cephadm:Fetching pub ssh key... 2024-03-29T04:45:58.149 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-03-29T04:45:58.149 DEBUG:teuthology.orchestra.run.smithi181:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-03-29T04:45:58.205 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-03-29T04:45:58.205 DEBUG:teuthology.orchestra.run.smithi181:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDkAcLiN5OL9FkxpFWSlDOwn3Q1VrmM/hLaKxLcXMMjyacEdsp/rIucDLq3/D5s4lB8riTXBQkVmlVkC82dAjIy/tkHcpbc4NEzsSt8M1G6iU8Jz/eGrBF+uqnILR3y0VkNHjafaYhxn/6awRArbVYBqOuvKw6TeAQVGModuk6rPldZjjrraRhU4F5Genf8SN79eCc+bwHYnbvV1J6HS/oJb8yacf06P1IpS6IY00m6y9YGiYHseRL2870pIahZIW/Nx1SV9W6I7IqxTNs2q8advdpTGfjsJndcVaaPsKM4yVo/zkwzeW+Sk5IZkOsZN8HmpvX1P+YiyxvlvK7gBigxO74WhfVC/CBn9H57n08ftGBPwOrYtLAytdUnKRaZE5RwacUUrj3Cqopdw5dBp22RDcFMh9i1IiiiRx172eirpHCh9HsveJFxdNvtLM1UibkHHLhIogoq26d5UkA9gDXr9CM/ySdD5x3n+ylum8Xl/Rs6F4ITtIxxxy9JO6InZq8= ceph-dfcecab8-ed86-11ee-95d0-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-29T04:45:58.283 INFO:teuthology.orchestra.run.smithi181.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDkAcLiN5OL9FkxpFWSlDOwn3Q1VrmM/hLaKxLcXMMjyacEdsp/rIucDLq3/D5s4lB8riTXBQkVmlVkC82dAjIy/tkHcpbc4NEzsSt8M1G6iU8Jz/eGrBF+uqnILR3y0VkNHjafaYhxn/6awRArbVYBqOuvKw6TeAQVGModuk6rPldZjjrraRhU4F5Genf8SN79eCc+bwHYnbvV1J6HS/oJb8yacf06P1IpS6IY00m6y9YGiYHseRL2870pIahZIW/Nx1SV9W6I7IqxTNs2q8advdpTGfjsJndcVaaPsKM4yVo/zkwzeW+Sk5IZkOsZN8HmpvX1P+YiyxvlvK7gBigxO74WhfVC/CBn9H57n08ftGBPwOrYtLAytdUnKRaZE5RwacUUrj3Cqopdw5dBp22RDcFMh9i1IiiiRx172eirpHCh9HsveJFxdNvtLM1UibkHHLhIogoq26d5UkA9gDXr9CM/ySdD5x3n+ylum8Xl/Rs6F4ITtIxxxy9JO6InZq8= ceph-dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:45:58.297 DEBUG:teuthology.orchestra.run.smithi187:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDkAcLiN5OL9FkxpFWSlDOwn3Q1VrmM/hLaKxLcXMMjyacEdsp/rIucDLq3/D5s4lB8riTXBQkVmlVkC82dAjIy/tkHcpbc4NEzsSt8M1G6iU8Jz/eGrBF+uqnILR3y0VkNHjafaYhxn/6awRArbVYBqOuvKw6TeAQVGModuk6rPldZjjrraRhU4F5Genf8SN79eCc+bwHYnbvV1J6HS/oJb8yacf06P1IpS6IY00m6y9YGiYHseRL2870pIahZIW/Nx1SV9W6I7IqxTNs2q8advdpTGfjsJndcVaaPsKM4yVo/zkwzeW+Sk5IZkOsZN8HmpvX1P+YiyxvlvK7gBigxO74WhfVC/CBn9H57n08ftGBPwOrYtLAytdUnKRaZE5RwacUUrj3Cqopdw5dBp22RDcFMh9i1IiiiRx172eirpHCh9HsveJFxdNvtLM1UibkHHLhIogoq26d5UkA9gDXr9CM/ySdD5x3n+ylum8Xl/Rs6F4ITtIxxxy9JO6InZq8= ceph-dfcecab8-ed86-11ee-95d0-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-29T04:45:58.342 INFO:teuthology.orchestra.run.smithi187.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDkAcLiN5OL9FkxpFWSlDOwn3Q1VrmM/hLaKxLcXMMjyacEdsp/rIucDLq3/D5s4lB8riTXBQkVmlVkC82dAjIy/tkHcpbc4NEzsSt8M1G6iU8Jz/eGrBF+uqnILR3y0VkNHjafaYhxn/6awRArbVYBqOuvKw6TeAQVGModuk6rPldZjjrraRhU4F5Genf8SN79eCc+bwHYnbvV1J6HS/oJb8yacf06P1IpS6IY00m6y9YGiYHseRL2870pIahZIW/Nx1SV9W6I7IqxTNs2q8advdpTGfjsJndcVaaPsKM4yVo/zkwzeW+Sk5IZkOsZN8HmpvX1P+YiyxvlvK7gBigxO74WhfVC/CBn9H57n08ftGBPwOrYtLAytdUnKRaZE5RwacUUrj3Cqopdw5dBp22RDcFMh9i1IiiiRx172eirpHCh9HsveJFxdNvtLM1UibkHHLhIogoq26d5UkA9gDXr9CM/ySdD5x3n+ylum8Xl/Rs6F4ITtIxxxy9JO6InZq8= ceph-dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:45:58.357 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-03-29T04:45:58.561 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:45:58 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/325273305' entity='client.admin' 2024-03-29T04:45:58.715 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:46:01.349 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:01 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:01.349 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:01 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:01.349 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:01 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi181", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:46:01.349 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:01 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:01.349 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:01 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:01.349 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:01 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:02.220 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-03-29T04:46:02.220 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-03-29T04:46:02.568 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:02 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1638015480' entity='client.admin' 2024-03-29T04:46:02.568 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:02 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:02.568 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:02 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:02.657 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:46:05.141 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi187 2024-03-29T04:46:05.141 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-29T04:46:05.142 DEBUG:teuthology.orchestra.run.smithi187:> dd of=/etc/ceph/ceph.conf 2024-03-29T04:46:05.164 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-29T04:46:05.164 DEBUG:teuthology.orchestra.run.smithi187:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:46:05.225 INFO:tasks.cephadm:Adding host smithi187 to orchestrator... 2024-03-29T04:46:05.226 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch host add smithi187 2024-03-29T04:46:05.481 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:46:05.681 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:05 smithi181 ceph-mon[29765]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:46:05.681 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:05 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:05.682 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:05 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:05.682 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:05 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:05.682 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:05 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:05.682 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:05 smithi181 ceph-mon[29765]: Updating smithi181:/etc/ceph/ceph.conf 2024-03-29T04:46:05.682 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:05 smithi181 ceph-mon[29765]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:46:05.682 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:05 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:05.682 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:05 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:05.682 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:05 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:06.530 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:06 smithi181 ceph-mon[29765]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:46:06.530 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:06 smithi181 ceph-mon[29765]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:46:08.098 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:07 smithi181 ceph-mon[29765]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi187", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:46:09.195 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:08 smithi181 ceph-mon[29765]: Deploying cephadm binary to smithi187 2024-03-29T04:46:09.234 INFO:teuthology.orchestra.run.smithi181.stdout:Added host 'smithi187' with addr '172.21.15.187' 2024-03-29T04:46:09.887 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch host ls --format=json 2024-03-29T04:46:10.137 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:46:10.581 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:10 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:10.581 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:10 smithi181 ceph-mon[29765]: Added host smithi187 2024-03-29T04:46:10.582 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:10 smithi181 ceph-mon[29765]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:10.582 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:10 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:10.582 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:10 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:10.582 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:10 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:11.794 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:46:11.794 INFO:teuthology.orchestra.run.smithi181.stdout:[{"addr": "172.21.15.181", "hostname": "smithi181", "labels": [], "status": ""}, {"addr": "172.21.15.187", "hostname": "smithi187", "labels": [], "status": ""}] 2024-03-29T04:46:11.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:11 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:12.349 INFO:tasks.cephadm:Setting crush tunables to default 2024-03-29T04:46:12.349 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd crush tunables default 2024-03-29T04:46:12.594 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:46:12.695 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:12 smithi181 ceph-mon[29765]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:13.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:13 smithi181 ceph-mon[29765]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:46:13.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:13 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:13.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:13 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:13.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:13 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:13.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:13 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:13.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:13 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi187", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:46:13.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:13 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:13.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:13 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:14.615 INFO:teuthology.orchestra.run.smithi181.stderr:adjusted tunables profile to default 2024-03-29T04:46:14.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:14 smithi181 ceph-mon[29765]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:14.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:14 smithi181 ceph-mon[29765]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:46:14.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:14 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3220421940' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-29T04:46:14.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:14 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:14.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:14 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:14.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:14 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:15.236 INFO:tasks.cephadm:Adding mon.a on smithi181 2024-03-29T04:46:15.236 INFO:tasks.cephadm:Adding mon.c on smithi181 2024-03-29T04:46:15.236 INFO:tasks.cephadm:Adding mon.b on smithi187 2024-03-29T04:46:15.237 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch apply mon '3;smithi181:172.21.15.181=a;smithi181:[v2:172.21.15.181:3301,v1:172.21.15.181:6790]=c;smithi187:172.21.15.187=b' 2024-03-29T04:46:15.477 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:46:15.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:15 smithi181 ceph-mon[29765]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:46:15.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:15 smithi181 ceph-mon[29765]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:46:15.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:15 smithi181 ceph-mon[29765]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:46:15.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:15 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3220421940' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-29T04:46:15.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:15 smithi181 ceph-mon[29765]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:46:16.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:16 smithi181 ceph-mon[29765]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:17.202 INFO:teuthology.orchestra.run.smithi187.stdout:Scheduled mon update... 2024-03-29T04:46:17.827 DEBUG:teuthology.orchestra.run.smithi181:mon.c> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.c.service 2024-03-29T04:46:17.830 DEBUG:teuthology.orchestra.run.smithi187:mon.b> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.b.service 2024-03-29T04:46:17.832 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-03-29T04:46:17.832 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-29T04:46:18.644 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:46:18.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:18 smithi181 ceph-mon[29765]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi181:172.21.15.181=a;smithi181:[v2:172.21.15.181:3301,v1:172.21.15.181:6790]=c;smithi187:172.21.15.187=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:46:18.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:18 smithi181 ceph-mon[29765]: Saving service mon spec with placement smithi181:172.21.15.181=a;smithi181:[v2:172.21.15.181:3301,v1:172.21.15.181:6790]=c;smithi187:172.21.15.187=b;count:3 2024-03-29T04:46:18.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:18 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:18.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:18 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:18.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:18 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:18.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:18 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:18.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:18 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:18.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:18 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:46:18.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:18 smithi181 ceph-mon[29765]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:18.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:18 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:18.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:18 smithi181 ceph-mon[29765]: Deploying daemon mon.b on smithi187 2024-03-29T04:46:20.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:20 smithi181 ceph-mon[29765]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:22.052 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:46:22.053 INFO:teuthology.orchestra.run.smithi187.stdout:{"epoch":1,"fsid":"dfcecab8-ed86-11ee-95d0-87774f69a715","modified":"2024-03-29T04:44:46.123593Z","created":"2024-03-29T04:44:46.123593Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:3300","nonce":0},{"type":"v1","addr":"172.21.15.181:6789","nonce":0}]},"addr":"172.21.15.181:6789/0","public_addr":"172.21.15.181:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-29T04:46:22.054 INFO:teuthology.orchestra.run.smithi187.stderr:dumped monmap epoch 1 2024-03-29T04:46:22.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:22 smithi181 ceph-mon[29765]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:22.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:22 smithi181 ceph-mon[29765]: from='client.? 172.21.15.187:0/2505523426' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:46:23.932 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-03-29T04:46:23.933 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-29T04:46:24.190 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:46:27.862 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:46:27.862 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: mon.a calling monitor election 2024-03-29T04:46:27.863 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:27.863 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:27.863 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:27.863 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:27.863 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: mon.b calling monitor election 2024-03-29T04:46:27.863 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:27.863 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:27.863 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:27.863 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:27.863 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-29T04:46:27.863 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: monmap e2: 2 mons at {a=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],b=[v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:46:27.863 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: fsmap 2024-03-29T04:46:27.863 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:46:27.863 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: mgrmap e13: y(active, since 38s) 2024-03-29T04:46:27.864 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: overall HEALTH_OK 2024-03-29T04:46:27.864 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:27.864 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:27.864 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:27.864 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:46:27.864 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:27 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:27.876 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:27 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:28.782 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:28 smithi187 ceph-mon[33595]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:28.782 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:28 smithi187 ceph-mon[33595]: Deploying daemon mon.c on smithi181 2024-03-29T04:46:28.782 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:28 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:28.800 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:28 smithi181 ceph-mon[29765]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:28.800 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:28 smithi181 ceph-mon[29765]: Deploying daemon mon.c on smithi181 2024-03-29T04:46:28.800 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:28 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:28.997 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:46:28.997 INFO:teuthology.orchestra.run.smithi187.stdout:{"epoch":2,"fsid":"dfcecab8-ed86-11ee-95d0-87774f69a715","modified":"2024-03-29T04:46:22.394992Z","created":"2024-03-29T04:44:46.123593Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:3300","nonce":0},{"type":"v1","addr":"172.21.15.181:6789","nonce":0}]},"addr":"172.21.15.181:6789/0","public_addr":"172.21.15.181:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:3300","nonce":0},{"type":"v1","addr":"172.21.15.187:6789","nonce":0}]},"addr":"172.21.15.187:6789/0","public_addr":"172.21.15.187:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-03-29T04:46:28.999 INFO:teuthology.orchestra.run.smithi187.stderr:dumped monmap epoch 2 2024-03-29T04:46:29.622 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:29 smithi187 ceph-mon[33595]: from='client.? 172.21.15.187:0/2703153986' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:46:29.672 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:29 smithi181 ceph-mon[29765]: from='client.? 172.21.15.187:0/2703153986' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:46:30.368 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 podman[41145]: 2024-03-29 04:46:30.241335103 +0000 UTC m=+0.122664431 container create 16379529f880470e0d1be0f5e1a7793dca0e4d97691cd19fce2c20f3043fca9e (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-03-29T04:46:30.646 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-03-29T04:46:30.646 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-29T04:46:30.876 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:30 smithi187 ceph-mon[33595]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:30.894 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:46:30.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[29765]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:30.946 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 podman[41145]: 2024-03-29 04:46:30.460187712 +0000 UTC m=+0.341516999 container init 16379529f880470e0d1be0f5e1a7793dca0e4d97691cd19fce2c20f3043fca9e (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-03-29T04:46:30.946 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 podman[41145]: 2024-03-29 04:46:30.465498334 +0000 UTC m=+0.346827620 container start 16379529f880470e0d1be0f5e1a7793dca0e4d97691cd19fce2c20f3043fca9e (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=quincy-4213e17, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-03-29T04:46:30.946 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: set uid:gid to 167:167 (ceph:ceph) 2024-03-29T04:46:30.946 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable), process ceph-mon, pid 7 2024-03-29T04:46:30.946 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: pidfile_write: ignore empty --pid-file 2024-03-29T04:46:30.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: load: jerasure load: lrc 2024-03-29T04:46:30.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: RocksDB version: 6.15.5 2024-03-29T04:46:30.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-03-29T04:46:30.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Compile date Mar 28 2024 2024-03-29T04:46:30.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: DB SUMMARY 2024-03-29T04:46:30.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: DB Session ID: LWV6WZ1YMG8EBNVEFV4T 2024-03-29T04:46:30.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: CURRENT file: CURRENT 2024-03-29T04:46:30.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: IDENTITY file: IDENTITY 2024-03-29T04:46:30.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-03-29T04:46:30.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-03-29T04:46:30.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-03-29T04:46:30.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.error_if_exists: 0 2024-03-29T04:46:30.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.create_if_missing: 0 2024-03-29T04:46:30.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.paranoid_checks: 1 2024-03-29T04:46:30.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-29T04:46:30.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.env: 0x55c20a846a80 2024-03-29T04:46:30.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.fs: Posix File System 2024-03-29T04:46:30.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.info_log: 0x55c20c142e80 2024-03-29T04:46:30.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_file_opening_threads: 16 2024-03-29T04:46:30.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.statistics: (nil) 2024-03-29T04:46:30.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.use_fsync: 0 2024-03-29T04:46:30.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_log_file_size: 0 2024-03-29T04:46:30.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-29T04:46:30.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-29T04:46:30.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.keep_log_file_num: 1000 2024-03-29T04:46:30.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.recycle_log_file_num: 0 2024-03-29T04:46:30.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.allow_fallocate: 1 2024-03-29T04:46:30.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.allow_mmap_reads: 0 2024-03-29T04:46:30.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.allow_mmap_writes: 0 2024-03-29T04:46:30.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.use_direct_reads: 0 2024-03-29T04:46:30.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-29T04:46:30.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.create_missing_column_families: 0 2024-03-29T04:46:30.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.db_log_dir: 2024-03-29T04:46:30.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-03-29T04:46:30.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-29T04:46:30.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-29T04:46:30.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-29T04:46:30.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-29T04:46:30.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-29T04:46:30.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-29T04:46:30.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.advise_random_on_open: 1 2024-03-29T04:46:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.db_write_buffer_size: 0 2024-03-29T04:46:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.write_buffer_manager: 0x55c20ceb2930 2024-03-29T04:46:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-29T04:46:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-03-29T04:46:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-29T04:46:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-29T04:46:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.rate_limiter: (nil) 2024-03-29T04:46:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-29T04:46:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.wal_recovery_mode: 2 2024-03-29T04:46:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.enable_thread_tracking: 0 2024-03-29T04:46:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.enable_pipelined_write: 0 2024-03-29T04:46:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.unordered_write: 0 2024-03-29T04:46:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-29T04:46:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-29T04:46:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-29T04:46:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-29T04:46:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.row_cache: None 2024-03-29T04:46:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.wal_filter: None 2024-03-29T04:46:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-29T04:46:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.allow_ingest_behind: 0 2024-03-29T04:46:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.preserve_deletes: 0 2024-03-29T04:46:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.two_write_queues: 0 2024-03-29T04:46:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.manual_wal_flush: 0 2024-03-29T04:46:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.atomic_flush: 0 2024-03-29T04:46:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-29T04:46:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-29T04:46:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-29T04:46:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.log_readahead_size: 0 2024-03-29T04:46:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-29T04:46:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.best_efforts_recovery: 0 2024-03-29T04:46:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-29T04:46:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-29T04:46:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.allow_data_in_errors: 0 2024-03-29T04:46:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.db_host_id: __hostname__ 2024-03-29T04:46:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_background_jobs: 2 2024-03-29T04:46:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_background_compactions: -1 2024-03-29T04:46:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_subcompactions: 1 2024-03-29T04:46:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-29T04:46:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-29T04:46:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-29T04:46:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_total_wal_size: 0 2024-03-29T04:46:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-29T04:46:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-29T04:46:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-29T04:46:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-29T04:46:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_open_files: -1 2024-03-29T04:46:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.bytes_per_sync: 0 2024-03-29T04:46:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-29T04:46:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-29T04:46:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compaction_readahead_size: 0 2024-03-29T04:46:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_background_flushes: -1 2024-03-29T04:46:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Compression algorithms supported: 2024-03-29T04:46:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-29T04:46:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: kZSTD supported: 0 2024-03-29T04:46:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: kXpressCompression supported: 0 2024-03-29T04:46:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: kLZ4HCCompression supported: 1 2024-03-29T04:46:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: kZlibCompression supported: 1 2024-03-29T04:46:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: kSnappyCompression supported: 1 2024-03-29T04:46:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: kLZ4Compression supported: 1 2024-03-29T04:46:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: kBZip2Compression supported: 0 2024-03-29T04:46:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-29T04:46:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-03-29T04:46:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-03-29T04:46:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-29T04:46:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.merge_operator: 2024-03-29T04:46:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compaction_filter: None 2024-03-29T04:46:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compaction_filter_factory: None 2024-03-29T04:46:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.sst_partitioner_factory: None 2024-03-29T04:46:30.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-29T04:46:30.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-29T04:46:30.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c20c192f58) 2024-03-29T04:46:30.957 INFO:journalctl@ceph.mon.c.smithi181.stdout: cache_index_and_filter_blocks: 1 2024-03-29T04:46:30.957 INFO:journalctl@ceph.mon.c.smithi181.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-29T04:46:30.957 INFO:journalctl@ceph.mon.c.smithi181.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-29T04:46:30.957 INFO:journalctl@ceph.mon.c.smithi181.stdout: pin_top_level_index_and_filter: 1 2024-03-29T04:46:30.957 INFO:journalctl@ceph.mon.c.smithi181.stdout: index_type: 0 2024-03-29T04:46:30.957 INFO:journalctl@ceph.mon.c.smithi181.stdout: data_block_index_type: 0 2024-03-29T04:46:30.957 INFO:journalctl@ceph.mon.c.smithi181.stdout: index_shortening: 1 2024-03-29T04:46:30.957 INFO:journalctl@ceph.mon.c.smithi181.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-29T04:46:30.957 INFO:journalctl@ceph.mon.c.smithi181.stdout: hash_index_allow_collision: 1 2024-03-29T04:46:30.957 INFO:journalctl@ceph.mon.c.smithi181.stdout: checksum: 1 2024-03-29T04:46:30.957 INFO:journalctl@ceph.mon.c.smithi181.stdout: no_block_cache: 0 2024-03-29T04:46:30.957 INFO:journalctl@ceph.mon.c.smithi181.stdout: block_cache: 0x55c20c1c5090 2024-03-29T04:46:30.958 INFO:journalctl@ceph.mon.c.smithi181.stdout: block_cache_name: BinnedLRUCache 2024-03-29T04:46:30.958 INFO:journalctl@ceph.mon.c.smithi181.stdout: block_cache_options: 2024-03-29T04:46:30.958 INFO:journalctl@ceph.mon.c.smithi181.stdout: capacity : 536870912 2024-03-29T04:46:30.958 INFO:journalctl@ceph.mon.c.smithi181.stdout: num_shard_bits : 4 2024-03-29T04:46:30.958 INFO:journalctl@ceph.mon.c.smithi181.stdout: strict_capacity_limit : 0 2024-03-29T04:46:30.958 INFO:journalctl@ceph.mon.c.smithi181.stdout: high_pri_pool_ratio: 0.000 2024-03-29T04:46:30.958 INFO:journalctl@ceph.mon.c.smithi181.stdout: block_cache_compressed: (nil) 2024-03-29T04:46:30.958 INFO:journalctl@ceph.mon.c.smithi181.stdout: persistent_cache: (nil) 2024-03-29T04:46:30.958 INFO:journalctl@ceph.mon.c.smithi181.stdout: block_size: 4096 2024-03-29T04:46:30.958 INFO:journalctl@ceph.mon.c.smithi181.stdout: block_size_deviation: 10 2024-03-29T04:46:30.958 INFO:journalctl@ceph.mon.c.smithi181.stdout: block_restart_interval: 16 2024-03-29T04:46:30.958 INFO:journalctl@ceph.mon.c.smithi181.stdout: index_block_restart_interval: 1 2024-03-29T04:46:30.958 INFO:journalctl@ceph.mon.c.smithi181.stdout: metadata_block_size: 4096 2024-03-29T04:46:30.958 INFO:journalctl@ceph.mon.c.smithi181.stdout: partition_filters: 0 2024-03-29T04:46:30.959 INFO:journalctl@ceph.mon.c.smithi181.stdout: use_delta_encoding: 1 2024-03-29T04:46:30.959 INFO:journalctl@ceph.mon.c.smithi181.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-03-29T04:46:30.959 INFO:journalctl@ceph.mon.c.smithi181.stdout: whole_key_filtering: 1 2024-03-29T04:46:30.959 INFO:journalctl@ceph.mon.c.smithi181.stdout: verify_compression: 0 2024-03-29T04:46:30.959 INFO:journalctl@ceph.mon.c.smithi181.stdout: read_amp_bytes_per_bit: 0 2024-03-29T04:46:30.959 INFO:journalctl@ceph.mon.c.smithi181.stdout: format_version: 4 2024-03-29T04:46:30.959 INFO:journalctl@ceph.mon.c.smithi181.stdout: enable_index_compression: 1 2024-03-29T04:46:30.959 INFO:journalctl@ceph.mon.c.smithi181.stdout: block_align: 0 2024-03-29T04:46:30.959 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.write_buffer_size: 33554432 2024-03-29T04:46:30.959 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_write_buffer_number: 2 2024-03-29T04:46:30.959 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compression: NoCompression 2024-03-29T04:46:30.959 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.bottommost_compression: Disabled 2024-03-29T04:46:30.959 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.prefix_extractor: nullptr 2024-03-29T04:46:30.959 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-29T04:46:30.959 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.num_levels: 7 2024-03-29T04:46:30.959 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-29T04:46:30.960 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-29T04:46:30.960 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-29T04:46:30.960 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-29T04:46:30.960 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-29T04:46:30.960 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-29T04:46:30.960 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-29T04:46:30.960 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:46:30.960 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-29T04:46:30.960 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-29T04:46:30.960 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-29T04:46:30.960 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compression_opts.level: 32767 2024-03-29T04:46:30.960 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compression_opts.strategy: 0 2024-03-29T04:46:30.960 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-29T04:46:30.960 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:46:30.960 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-29T04:46:30.960 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compression_opts.enabled: false 2024-03-29T04:46:30.961 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-29T04:46:30.961 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-29T04:46:30.961 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-29T04:46:30.961 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.target_file_size_base: 67108864 2024-03-29T04:46:30.961 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-29T04:46:30.961 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-29T04:46:30.961 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-29T04:46:30.961 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-29T04:46:30.961 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-29T04:46:30.961 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-29T04:46:30.961 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-29T04:46:30.961 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-29T04:46:30.961 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-29T04:46:30.961 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-29T04:46:30.961 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-29T04:46:30.961 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-29T04:46:30.962 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-29T04:46:30.962 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.arena_block_size: 4194304 2024-03-29T04:46:30.962 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-29T04:46:30.962 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-29T04:46:30.962 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-03-29T04:46:30.962 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.disable_auto_compactions: 0 2024-03-29T04:46:30.962 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-29T04:46:30.962 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-29T04:46:30.962 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-29T04:46:30.962 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-29T04:46:30.962 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-29T04:46:30.962 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-29T04:46:30.963 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-29T04:46:30.963 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-29T04:46:30.963 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-29T04:46:30.963 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-29T04:46:30.963 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.table_properties_collectors: 2024-03-29T04:46:30.963 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.inplace_update_support: 0 2024-03-29T04:46:30.963 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-29T04:46:30.963 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-29T04:46:30.963 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-29T04:46:30.963 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-29T04:46:30.963 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.bloom_locality: 0 2024-03-29T04:46:30.963 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.max_successive_merges: 0 2024-03-29T04:46:30.963 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-29T04:46:30.963 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.paranoid_file_checks: 0 2024-03-29T04:46:30.963 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.force_consistency_checks: 1 2024-03-29T04:46:30.964 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.report_bg_io_stats: 0 2024-03-29T04:46:30.964 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.ttl: 2592000 2024-03-29T04:46:30.964 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-29T04:46:30.964 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.enable_blob_files: false 2024-03-29T04:46:30.964 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.min_blob_size: 0 2024-03-29T04:46:30.964 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.blob_file_size: 268435456 2024-03-29T04:46:30.964 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-29T04:46:30.964 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-29T04:46:30.964 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-29T04:46:30.964 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-29T04:46:30.964 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-03-29T04:46:30.964 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-03-29T04:46:30.964 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687590492706, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687590493085, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1711687590, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "43b1cc4c-c43f-45ef-8c9b-08bd59041353", "db_session_id": "LWV6WZ1YMG8EBNVEFV4T"}} 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687590493390, "job": 1, "event": "recovery_finished"} 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55c20c159180 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: DB pointer 0x55c20c22e000 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout: ** DB Stats ** 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-29T04:46:30.965 INFO:journalctl@ceph.mon.c.smithi181.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-03-29T04:46:30.966 INFO:journalctl@ceph.mon.c.smithi181.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:46:30.966 INFO:journalctl@ceph.mon.c.smithi181.stdout: 2024-03-29T04:46:30.966 INFO:journalctl@ceph.mon.c.smithi181.stdout: ** Compaction Stats [default] ** 2024-03-29T04:46:30.966 INFO:journalctl@ceph.mon.c.smithi181.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:46:30.966 INFO:journalctl@ceph.mon.c.smithi181.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:46:30.966 INFO:journalctl@ceph.mon.c.smithi181.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.4 0.00 0.00 1 0.000 0 0 2024-03-29T04:46:30.966 INFO:journalctl@ceph.mon.c.smithi181.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.4 0.00 0.00 1 0.000 0 0 2024-03-29T04:46:30.966 INFO:journalctl@ceph.mon.c.smithi181.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.4 0.00 0.00 1 0.000 0 0 2024-03-29T04:46:30.966 INFO:journalctl@ceph.mon.c.smithi181.stdout: 2024-03-29T04:46:30.966 INFO:journalctl@ceph.mon.c.smithi181.stdout: ** Compaction Stats [default] ** 2024-03-29T04:46:30.966 INFO:journalctl@ceph.mon.c.smithi181.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:46:30.966 INFO:journalctl@ceph.mon.c.smithi181.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:46:30.966 INFO:journalctl@ceph.mon.c.smithi181.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.4 0.00 0.00 1 0.000 0 0 2024-03-29T04:46:30.966 INFO:journalctl@ceph.mon.c.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:46:30.966 INFO:journalctl@ceph.mon.c.smithi181.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: Cumulative compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: Interval compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: ** Compaction Stats [default] ** 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.4 0.00 0.00 1 0.000 0 0 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.4 0.00 0.00 1 0.000 0 0 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-03-29T04:46:30.967 INFO:journalctl@ceph.mon.c.smithi181.stdout: 2024-03-29T04:46:30.968 INFO:journalctl@ceph.mon.c.smithi181.stdout: ** Compaction Stats [default] ** 2024-03-29T04:46:30.968 INFO:journalctl@ceph.mon.c.smithi181.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:46:30.968 INFO:journalctl@ceph.mon.c.smithi181.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:46:30.968 INFO:journalctl@ceph.mon.c.smithi181.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.4 0.00 0.00 1 0.000 0 0 2024-03-29T04:46:30.968 INFO:journalctl@ceph.mon.c.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:46:30.968 INFO:journalctl@ceph.mon.c.smithi181.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-29T04:46:30.968 INFO:journalctl@ceph.mon.c.smithi181.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:46:30.968 INFO:journalctl@ceph.mon.c.smithi181.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:46:30.968 INFO:journalctl@ceph.mon.c.smithi181.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:46:30.968 INFO:journalctl@ceph.mon.c.smithi181.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:46:30.968 INFO:journalctl@ceph.mon.c.smithi181.stdout: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:46:30.968 INFO:journalctl@ceph.mon.c.smithi181.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:46:30.968 INFO:journalctl@ceph.mon.c.smithi181.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:46:30.968 INFO:journalctl@ceph.mon.c.smithi181.stdout: 2024-03-29T04:46:30.969 INFO:journalctl@ceph.mon.c.smithi181.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:46:30.969 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-03-29T04:46:30.969 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: using public_addrv [v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0] 2024-03-29T04:46:30.969 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: starting mon.c rank -1 at public addrs [v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0] at bind addrs [v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:46:30.969 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.c@-1(???) e0 preinit fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:46:30.969 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.c@-1(synchronizing).mds e1 new map 2024-03-29T04:46:30.969 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.c@-1(synchronizing).mds e1 print_map 2024-03-29T04:46:30.969 INFO:journalctl@ceph.mon.c.smithi181.stdout: e1 2024-03-29T04:46:30.969 INFO:journalctl@ceph.mon.c.smithi181.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-29T04:46:30.969 INFO:journalctl@ceph.mon.c.smithi181.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:46:30.969 INFO:journalctl@ceph.mon.c.smithi181.stdout: legacy client fscid: -1 2024-03-29T04:46:30.969 INFO:journalctl@ceph.mon.c.smithi181.stdout: 2024-03-29T04:46:30.969 INFO:journalctl@ceph.mon.c.smithi181.stdout: No filesystems configured 2024-03-29T04:46:30.969 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-03-29T04:46:30.969 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-03-29T04:46:30.970 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-03-29T04:46:30.970 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-03-29T04:46:30.970 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-03-29T04:46:30.970 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-03-29T04:46:30.970 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-29T04:46:30.970 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:46:30.970 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:46:30.970 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:46:30.970 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:30.970 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:46:30.970 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:30.970 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:30.970 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:30.971 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:30.971 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi187", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:46:30.971 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:30.971 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:30.971 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:30.971 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:46:30.971 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3220421940' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-29T04:46:30.971 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:30.971 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:30.971 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:30.971 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:46:30.971 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:46:30.971 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:46:30.971 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3220421940' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-29T04:46:30.971 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:46:30.971 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:30.972 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi181:172.21.15.181=a;smithi181:[v2:172.21.15.181:3301,v1:172.21.15.181:6790]=c;smithi187:172.21.15.187=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:46:30.972 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: Saving service mon spec with placement smithi181:172.21.15.181=a;smithi181:[v2:172.21.15.181:3301,v1:172.21.15.181:6790]=c;smithi187:172.21.15.187=b;count:3 2024-03-29T04:46:30.972 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:30.972 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:30.972 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:30.972 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:30.972 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:30.972 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:46:30.972 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:30.972 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:30.972 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: Deploying daemon mon.b on smithi187 2024-03-29T04:46:30.972 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:30.972 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:30.973 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='client.? 172.21.15.187:0/2505523426' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:46:30.973 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:46:30.973 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.a calling monitor election 2024-03-29T04:46:30.973 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:30.973 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:30.973 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:30.973 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:30.973 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.b calling monitor election 2024-03-29T04:46:30.973 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:30.973 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:30.973 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:30.973 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:30.974 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-29T04:46:30.974 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: monmap e2: 2 mons at {a=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],b=[v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:46:30.974 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: fsmap 2024-03-29T04:46:30.974 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:46:30.974 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mgrmap e13: y(active, since 38s) 2024-03-29T04:46:30.974 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: overall HEALTH_OK 2024-03-29T04:46:30.974 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:30.974 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:30.974 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:30.974 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:46:30.974 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:30.974 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:30.974 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: Deploying daemon mon.c on smithi181 2024-03-29T04:46:30.974 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:30.975 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: from='client.? 172.21.15.187:0/2703153986' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:46:30.975 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:30.975 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 ceph-mon[41183]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-03-29T04:46:30.975 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 bash[41145]: 16379529f880470e0d1be0f5e1a7793dca0e4d97691cd19fce2c20f3043fca9e 2024-03-29T04:46:30.975 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:30 smithi181 systemd[1]: Started Ceph mon.c for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:46:35.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:46:35.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: mon.a calling monitor election 2024-03-29T04:46:35.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:35.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: mon.b calling monitor election 2024-03-29T04:46:35.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:35.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: mon.c calling monitor election 2024-03-29T04:46:35.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:35.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:46:35.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: monmap e3: 3 mons at {a=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],b=[v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0],c=[v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:46:35.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: fsmap 2024-03-29T04:46:35.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:46:35.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: mgrmap e13: y(active, since 46s) 2024-03-29T04:46:35.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: overall HEALTH_OK 2024-03-29T04:46:35.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:35.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:35.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:35.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:35 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:35.940 INFO:teuthology.orchestra.run.smithi187.stdout: 2024-03-29T04:46:35.940 INFO:teuthology.orchestra.run.smithi187.stdout:{"epoch":3,"fsid":"dfcecab8-ed86-11ee-95d0-87774f69a715","modified":"2024-03-29T04:46:30.517496Z","created":"2024-03-29T04:44:46.123593Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:3300","nonce":0},{"type":"v1","addr":"172.21.15.181:6789","nonce":0}]},"addr":"172.21.15.181:6789/0","public_addr":"172.21.15.181:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:3300","nonce":0},{"type":"v1","addr":"172.21.15.187:6789","nonce":0}]},"addr":"172.21.15.187:6789/0","public_addr":"172.21.15.187:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:3301","nonce":0},{"type":"v1","addr":"172.21.15.181:6790","nonce":0}]},"addr":"172.21.15.181:6790/0","public_addr":"172.21.15.181:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-03-29T04:46:35.943 INFO:teuthology.orchestra.run.smithi187.stderr:dumped monmap epoch 3 2024-03-29T04:46:35.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:46:35.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: mon.a calling monitor election 2024-03-29T04:46:35.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:35.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: mon.b calling monitor election 2024-03-29T04:46:35.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:35.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: mon.c calling monitor election 2024-03-29T04:46:35.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:35.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:46:35.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: monmap e3: 3 mons at {a=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],b=[v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0],c=[v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:46:35.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: fsmap 2024-03-29T04:46:35.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:46:35.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: mgrmap e13: y(active, since 46s) 2024-03-29T04:46:35.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: overall HEALTH_OK 2024-03-29T04:46:35.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:35.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:35.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:35.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:35.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:46:35.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: mon.a calling monitor election 2024-03-29T04:46:35.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:46:35.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: mon.b calling monitor election 2024-03-29T04:46:35.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:35.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: mon.c calling monitor election 2024-03-29T04:46:35.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:35.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:35.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:46:35.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: monmap e3: 3 mons at {a=[v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0],b=[v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0],c=[v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:46:35.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: fsmap 2024-03-29T04:46:35.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:46:35.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: mgrmap e13: y(active, since 46s) 2024-03-29T04:46:35.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: overall HEALTH_OK 2024-03-29T04:46:35.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:35.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:35.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:35.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:35 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:36.524 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-03-29T04:46:36.524 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph config generate-minimal-conf 2024-03-29T04:46:36.810 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:46:36.876 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:36 smithi187 ceph-mon[33595]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:36.876 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:36 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:36.876 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:36 smithi187 ceph-mon[33595]: from='client.? 172.21.15.187:0/165737393' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:46:36.876 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:36 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:36.933 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:36 smithi181 ceph-mon[41183]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:36.933 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:36 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:36.933 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:36 smithi181 ceph-mon[41183]: from='client.? 172.21.15.187:0/165737393' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:46:36.933 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:36 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:36.934 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:36 smithi181 ceph-mon[29765]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:36.934 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:36 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:36.934 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:36 smithi181 ceph-mon[29765]: from='client.? 172.21.15.187:0/165737393' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:46:36.934 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:36 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:46:37.946 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:37.946 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:37.946 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[41183]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:37.946 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:37.946 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:37.946 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:37.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:37.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[41183]: Updating smithi181:/etc/ceph/ceph.conf 2024-03-29T04:46:37.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[41183]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:46:37.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[41183]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:46:37.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[41183]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:46:37.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:37.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:37.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:37.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:37.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[29765]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:37.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:37.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:37.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:37.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:37.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[29765]: Updating smithi181:/etc/ceph/ceph.conf 2024-03-29T04:46:37.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[29765]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:46:37.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[29765]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:46:37.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[29765]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:46:37.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:37.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:38.376 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:38.376 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:38.376 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:37 smithi187 ceph-mon[33595]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:38.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:38.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:38.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:38.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:38.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:37 smithi187 ceph-mon[33595]: Updating smithi181:/etc/ceph/ceph.conf 2024-03-29T04:46:38.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:37 smithi187 ceph-mon[33595]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:46:38.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:37 smithi187 ceph-mon[33595]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:46:38.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:37 smithi187 ceph-mon[33595]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:46:38.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:38.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:38.655 INFO:teuthology.orchestra.run.smithi181.stdout:# minimal ceph.conf for dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:46:38.655 INFO:teuthology.orchestra.run.smithi181.stdout:[global] 2024-03-29T04:46:38.655 INFO:teuthology.orchestra.run.smithi181.stdout: fsid = dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:46:38.656 INFO:teuthology.orchestra.run.smithi181.stdout: mon_host = [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] [v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0] 2024-03-29T04:46:39.196 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:39.196 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:39.196 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:39.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[41183]: Reconfiguring mon.a (unknown last config time)... 2024-03-29T04:46:39.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:46:39.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:46:39.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:39.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[41183]: Reconfiguring daemon mon.a on smithi181 2024-03-29T04:46:39.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1794168870' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:39.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:39.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:39.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:39.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[29765]: Reconfiguring mon.a (unknown last config time)... 2024-03-29T04:46:39.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:46:39.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:46:39.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:39.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[29765]: Reconfiguring daemon mon.a on smithi181 2024-03-29T04:46:39.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:38 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1794168870' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:39.376 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:38 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:39.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:38 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:39.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:38 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:39.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:38 smithi187 ceph-mon[33595]: Reconfiguring mon.a (unknown last config time)... 2024-03-29T04:46:39.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:38 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:46:39.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:38 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:46:39.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:38 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:39.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:38 smithi187 ceph-mon[33595]: Reconfiguring daemon mon.a on smithi181 2024-03-29T04:46:39.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:38 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1794168870' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:40.066 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-03-29T04:46:40.066 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-03-29T04:46:40.066 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/ceph/ceph.conf 2024-03-29T04:46:40.099 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-03-29T04:46:40.099 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:46:40.167 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-29T04:46:40.167 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd of=/etc/ceph/ceph.conf 2024-03-29T04:46:40.198 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-29T04:46:40.198 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:46:40.267 INFO:tasks.cephadm:Adding mgr.y on smithi181 2024-03-29T04:46:40.267 INFO:tasks.cephadm:Adding mgr.x on smithi187 2024-03-29T04:46:40.267 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch apply mgr '2;smithi181=y;smithi187=x' 2024-03-29T04:46:40.332 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:40 smithi187 ceph-mon[33595]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:40.332 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:40.332 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:40.332 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:40.332 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:40 smithi187 ceph-mon[33595]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:46:40.332 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:46:40.333 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:46:40.333 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:40.333 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:40 smithi187 ceph-mon[33595]: Reconfiguring daemon mon.c on smithi181 2024-03-29T04:46:40.546 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[29765]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:40.546 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:40.546 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:40.546 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:40.547 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[29765]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:46:40.547 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:46:40.547 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:46:40.547 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:40.547 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[29765]: Reconfiguring daemon mon.c on smithi181 2024-03-29T04:46:40.547 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[41183]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:40.547 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:40.547 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:40.547 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:40.547 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[41183]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:46:40.547 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:46:40.547 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:46:40.547 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:40.547 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:40 smithi181 ceph-mon[41183]: Reconfiguring daemon mon.c on smithi181 2024-03-29T04:46:40.552 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:46:42.376 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:42 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:42.376 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:42 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:42.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:42 smithi187 ceph-mon[33595]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:46:42.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:42 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:46:42.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:42 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:46:42.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:42 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:42.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:42 smithi187 ceph-mon[33595]: Reconfiguring daemon mon.b on smithi187 2024-03-29T04:46:42.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:42 smithi187 ceph-mon[33595]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:42.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:42 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:42.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:42 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:42.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:42 smithi181 ceph-mon[41183]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:46:42.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:42 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:46:42.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:42 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:46:42.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:42 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:42.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:42 smithi181 ceph-mon[41183]: Reconfiguring daemon mon.b on smithi187 2024-03-29T04:46:42.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:42 smithi181 ceph-mon[41183]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:42.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:42 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:42.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:42 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:42.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:42 smithi181 ceph-mon[29765]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:46:42.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:42 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:46:42.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:42 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:46:42.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:42 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:42.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:42 smithi181 ceph-mon[29765]: Reconfiguring daemon mon.b on smithi187 2024-03-29T04:46:42.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:42 smithi181 ceph-mon[29765]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:42.750 INFO:teuthology.orchestra.run.smithi187.stdout:Scheduled mgr update... 2024-03-29T04:46:43.839 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:43 smithi187 ceph-mon[33595]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi181=y;smithi187=x", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:46:43.839 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:43 smithi187 ceph-mon[33595]: Saving service mgr spec with placement smithi181=y;smithi187=x;count:2 2024-03-29T04:46:43.840 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:43 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:43.840 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:43 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:43.840 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:43 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:43.840 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:43 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:43.840 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:43 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:43.840 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:43 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:43.840 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:43 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:43.840 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:43 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:46:43.840 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:43 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-29T04:46:43.840 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:43 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:46:43.840 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:43 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:43.863 DEBUG:teuthology.orchestra.run.smithi187:mgr.x> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.x.service 2024-03-29T04:46:43.906 INFO:tasks.cephadm:Deploying OSDs... 2024-03-29T04:46:43.907 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-03-29T04:46:43.907 DEBUG:teuthology.orchestra.run.smithi181:> dd if=/scratch_devs of=/dev/stdout 2024-03-29T04:46:43.926 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-29T04:46:43.926 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_1 2024-03-29T04:46:43.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[29765]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi181=y;smithi187=x", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:46:43.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[29765]: Saving service mgr spec with placement smithi181=y;smithi187=x;count:2 2024-03-29T04:46:43.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:43.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:43.946 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:43.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:43.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:43.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:43.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:43.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:46:43.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-29T04:46:43.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:46:43.947 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:43.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[41183]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi181=y;smithi187=x", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:46:43.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[41183]: Saving service mgr spec with placement smithi181=y;smithi187=x;count:2 2024-03-29T04:46:43.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:43.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:43.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:43.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:43.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:43.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:43.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:43.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:46:43.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-29T04:46:43.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:46:43.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:43 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:43.966 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-29T04:46:43.966 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:46:43.966 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-03-29T04:46:43.966 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:46:43.966 INFO:teuthology.orchestra.run.smithi181.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:46:43.966 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2024-03-29 04:46:29.960646248 +0000 2024-03-29T04:46:43.966 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2024-03-29 04:33:43.443940777 +0000 2024-03-29T04:46:43.967 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2024-03-29 04:33:43.443940777 +0000 2024-03-29T04:46:43.967 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: 2024-03-29 04:33:43.443940777 +0000 2024-03-29T04:46:43.967 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-29T04:46:44.038 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2024-03-29T04:46:44.038 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2024-03-29T04:46:44.038 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000351594 s, 1.5 MB/s 2024-03-29T04:46:44.040 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-29T04:46:44.097 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_2 2024-03-29T04:46:44.154 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-29T04:46:44.154 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:46:44.154 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-03-29T04:46:44.154 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:46:44.154 INFO:teuthology.orchestra.run.smithi181.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:46:44.154 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2024-03-29 04:46:29.961646228 +0000 2024-03-29T04:46:44.154 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2024-03-29 04:33:43.779934202 +0000 2024-03-29T04:46:44.154 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2024-03-29 04:33:43.779934202 +0000 2024-03-29T04:46:44.154 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: 2024-03-29 04:33:43.779934202 +0000 2024-03-29T04:46:44.154 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-29T04:46:44.221 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2024-03-29T04:46:44.221 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2024-03-29T04:46:44.221 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000248136 s, 2.1 MB/s 2024-03-29T04:46:44.222 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-29T04:46:44.279 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_3 2024-03-29T04:46:44.335 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-29T04:46:44.335 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:46:44.336 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-03-29T04:46:44.336 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:46:44.336 INFO:teuthology.orchestra.run.smithi181.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:46:44.336 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2024-03-29 04:46:29.961646228 +0000 2024-03-29T04:46:44.336 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2024-03-29 04:33:44.116927606 +0000 2024-03-29T04:46:44.336 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2024-03-29 04:33:44.116927606 +0000 2024-03-29T04:46:44.336 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: 2024-03-29 04:33:44.116927606 +0000 2024-03-29T04:46:44.336 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-29T04:46:44.402 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2024-03-29T04:46:44.402 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2024-03-29T04:46:44.402 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000195948 s, 2.6 MB/s 2024-03-29T04:46:44.404 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-29T04:46:44.459 DEBUG:teuthology.orchestra.run.smithi181:> stat /dev/vg_nvme/lv_4 2024-03-29T04:46:44.515 INFO:teuthology.orchestra.run.smithi181.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-29T04:46:44.515 INFO:teuthology.orchestra.run.smithi181.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:46:44.515 INFO:teuthology.orchestra.run.smithi181.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-03-29T04:46:44.515 INFO:teuthology.orchestra.run.smithi181.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:46:44.515 INFO:teuthology.orchestra.run.smithi181.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:46:44.516 INFO:teuthology.orchestra.run.smithi181.stdout:Access: 2024-03-29 04:46:29.961646228 +0000 2024-03-29T04:46:44.516 INFO:teuthology.orchestra.run.smithi181.stdout:Modify: 2024-03-29 04:33:44.456920952 +0000 2024-03-29T04:46:44.516 INFO:teuthology.orchestra.run.smithi181.stdout:Change: 2024-03-29 04:33:44.456920952 +0000 2024-03-29T04:46:44.516 INFO:teuthology.orchestra.run.smithi181.stdout: Birth: 2024-03-29 04:33:44.456920952 +0000 2024-03-29T04:46:44.516 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-29T04:46:44.581 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records in 2024-03-29T04:46:44.581 INFO:teuthology.orchestra.run.smithi181.stderr:1+0 records out 2024-03-29T04:46:44.581 INFO:teuthology.orchestra.run.smithi181.stderr:512 bytes copied, 0.000197678 s, 2.6 MB/s 2024-03-29T04:46:44.582 DEBUG:teuthology.orchestra.run.smithi181:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-29T04:46:44.638 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-29T04:46:44.638 DEBUG:teuthology.orchestra.run.smithi187:> dd if=/scratch_devs of=/dev/stdout 2024-03-29T04:46:44.655 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-29T04:46:44.655 DEBUG:teuthology.orchestra.run.smithi187:> stat /dev/vg_nvme/lv_1 2024-03-29T04:46:44.696 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:44 smithi181 ceph-mon[41183]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:44.696 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:44 smithi181 ceph-mon[41183]: Deploying daemon mgr.x on smithi187 2024-03-29T04:46:44.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:44 smithi181 ceph-mon[29765]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:44.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:44 smithi181 ceph-mon[29765]: Deploying daemon mgr.x on smithi187 2024-03-29T04:46:44.711 INFO:teuthology.orchestra.run.smithi187.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-29T04:46:44.711 INFO:teuthology.orchestra.run.smithi187.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:46:44.711 INFO:teuthology.orchestra.run.smithi187.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-03-29T04:46:44.711 INFO:teuthology.orchestra.run.smithi187.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:46:44.711 INFO:teuthology.orchestra.run.smithi187.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:46:44.711 INFO:teuthology.orchestra.run.smithi187.stdout:Access: 2024-03-29 04:46:21.654097783 +0000 2024-03-29T04:46:44.711 INFO:teuthology.orchestra.run.smithi187.stdout:Modify: 2024-03-29 04:29:24.630781983 +0000 2024-03-29T04:46:44.711 INFO:teuthology.orchestra.run.smithi187.stdout:Change: 2024-03-29 04:29:24.630781983 +0000 2024-03-29T04:46:44.711 INFO:teuthology.orchestra.run.smithi187.stdout: Birth: 2024-03-29 04:29:24.630781983 +0000 2024-03-29T04:46:44.711 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-29T04:46:44.778 INFO:teuthology.orchestra.run.smithi187.stderr:1+0 records in 2024-03-29T04:46:44.778 INFO:teuthology.orchestra.run.smithi187.stderr:1+0 records out 2024-03-29T04:46:44.778 INFO:teuthology.orchestra.run.smithi187.stderr:512 bytes copied, 0.000221774 s, 2.3 MB/s 2024-03-29T04:46:44.779 DEBUG:teuthology.orchestra.run.smithi187:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-29T04:46:44.835 DEBUG:teuthology.orchestra.run.smithi187:> stat /dev/vg_nvme/lv_2 2024-03-29T04:46:44.889 INFO:teuthology.orchestra.run.smithi187.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-29T04:46:44.890 INFO:teuthology.orchestra.run.smithi187.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:46:44.890 INFO:teuthology.orchestra.run.smithi187.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-03-29T04:46:44.890 INFO:teuthology.orchestra.run.smithi187.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:46:44.890 INFO:teuthology.orchestra.run.smithi187.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:46:44.890 INFO:teuthology.orchestra.run.smithi187.stdout:Access: 2024-03-29 04:46:21.654097783 +0000 2024-03-29T04:46:44.890 INFO:teuthology.orchestra.run.smithi187.stdout:Modify: 2024-03-29 04:29:24.964776412 +0000 2024-03-29T04:46:44.890 INFO:teuthology.orchestra.run.smithi187.stdout:Change: 2024-03-29 04:29:24.964776412 +0000 2024-03-29T04:46:44.890 INFO:teuthology.orchestra.run.smithi187.stdout: Birth: 2024-03-29 04:29:24.964776412 +0000 2024-03-29T04:46:44.890 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-29T04:46:44.952 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:44 smithi187 ceph-mon[33595]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:44.952 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:44 smithi187 ceph-mon[33595]: Deploying daemon mgr.x on smithi187 2024-03-29T04:46:44.954 INFO:teuthology.orchestra.run.smithi187.stderr:1+0 records in 2024-03-29T04:46:44.954 INFO:teuthology.orchestra.run.smithi187.stderr:1+0 records out 2024-03-29T04:46:44.955 INFO:teuthology.orchestra.run.smithi187.stderr:512 bytes copied, 0.000195806 s, 2.6 MB/s 2024-03-29T04:46:44.955 DEBUG:teuthology.orchestra.run.smithi187:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-29T04:46:45.010 DEBUG:teuthology.orchestra.run.smithi187:> stat /dev/vg_nvme/lv_3 2024-03-29T04:46:45.065 INFO:teuthology.orchestra.run.smithi187.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-29T04:46:45.065 INFO:teuthology.orchestra.run.smithi187.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:46:45.066 INFO:teuthology.orchestra.run.smithi187.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-03-29T04:46:45.066 INFO:teuthology.orchestra.run.smithi187.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:46:45.066 INFO:teuthology.orchestra.run.smithi187.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:46:45.066 INFO:teuthology.orchestra.run.smithi187.stdout:Access: 2024-03-29 04:46:21.654097783 +0000 2024-03-29T04:46:45.066 INFO:teuthology.orchestra.run.smithi187.stdout:Modify: 2024-03-29 04:29:25.295770891 +0000 2024-03-29T04:46:45.066 INFO:teuthology.orchestra.run.smithi187.stdout:Change: 2024-03-29 04:29:25.295770891 +0000 2024-03-29T04:46:45.066 INFO:teuthology.orchestra.run.smithi187.stdout: Birth: 2024-03-29 04:29:25.295770891 +0000 2024-03-29T04:46:45.066 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-29T04:46:45.257 INFO:teuthology.orchestra.run.smithi187.stderr:1+0 records in 2024-03-29T04:46:45.257 INFO:teuthology.orchestra.run.smithi187.stderr:1+0 records out 2024-03-29T04:46:45.257 INFO:teuthology.orchestra.run.smithi187.stderr:512 bytes copied, 0.000209982 s, 2.4 MB/s 2024-03-29T04:46:45.258 DEBUG:teuthology.orchestra.run.smithi187:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-29T04:46:45.273 DEBUG:teuthology.orchestra.run.smithi187:> stat /dev/vg_nvme/lv_4 2024-03-29T04:46:45.333 INFO:teuthology.orchestra.run.smithi187.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-29T04:46:45.334 INFO:teuthology.orchestra.run.smithi187.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:46:45.334 INFO:teuthology.orchestra.run.smithi187.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-03-29T04:46:45.334 INFO:teuthology.orchestra.run.smithi187.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:46:45.334 INFO:teuthology.orchestra.run.smithi187.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:46:45.334 INFO:teuthology.orchestra.run.smithi187.stdout:Access: 2024-03-29 04:46:45.214697529 +0000 2024-03-29T04:46:45.334 INFO:teuthology.orchestra.run.smithi187.stdout:Modify: 2024-03-29 04:29:25.634765237 +0000 2024-03-29T04:46:45.334 INFO:teuthology.orchestra.run.smithi187.stdout:Change: 2024-03-29 04:29:25.634765237 +0000 2024-03-29T04:46:45.334 INFO:teuthology.orchestra.run.smithi187.stdout: Birth: 2024-03-29 04:29:25.634765237 +0000 2024-03-29T04:46:45.334 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-29T04:46:45.439 INFO:teuthology.orchestra.run.smithi187.stderr:1+0 records in 2024-03-29T04:46:45.439 INFO:teuthology.orchestra.run.smithi187.stderr:1+0 records out 2024-03-29T04:46:45.439 INFO:teuthology.orchestra.run.smithi187.stderr:512 bytes copied, 0.000210043 s, 2.4 MB/s 2024-03-29T04:46:45.441 DEBUG:teuthology.orchestra.run.smithi187:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-29T04:46:45.455 INFO:tasks.cephadm:Deploying osd.0 on smithi181 with /dev/vg_nvme/lv_4... 2024-03-29T04:46:45.455 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-29T04:46:45.626 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:45 smithi187 systemd[1]: Starting Ceph mgr.x for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:46:45.711 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:46:45.950 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:45 smithi187 podman[35671]: 2024-03-29 04:46:45.67274592 +0000 UTC m=+0.105339724 container create f668da78a625b0b98ff26572a171402156e141a374d4ad282a9ff9f2cba44cc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2) 2024-03-29T04:46:45.950 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:45 smithi187 podman[35671]: 2024-03-29 04:46:45.584395645 +0000 UTC m=+0.016989421 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b 2024-03-29T04:46:45.950 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:45 smithi187 podman[35671]: 2024-03-29 04:46:45.858201873 +0000 UTC m=+0.290795642 container init f668da78a625b0b98ff26572a171402156e141a374d4ad282a9ff9f2cba44cc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-03-29T04:46:45.950 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:45 smithi187 podman[35671]: 2024-03-29 04:46:45.863867835 +0000 UTC m=+0.296461595 container start f668da78a625b0b98ff26572a171402156e141a374d4ad282a9ff9f2cba44cc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, RELEASE=quincy-4213e17, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-03-29T04:46:45.950 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:45 smithi187 ceph-mgr[35709]: -- 172.21.15.187:0/1180339225 <== mon.1 v2:172.21.15.187:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5607144789c0 con 0x56071444f000 2024-03-29T04:46:45.950 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:45 smithi187 bash[35671]: f668da78a625b0b98ff26572a171402156e141a374d4ad282a9ff9f2cba44cc8 2024-03-29T04:46:46.376 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:45 smithi187 systemd[1]: Started Ceph mgr.x for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:46:46.648 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:46 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:46.393+0000 7f6406c421c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:46:46.648 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:46 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:46.492+0000 7f6406c421c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:46:46.935 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:46 smithi187 ceph-mon[33595]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:46.935 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:46 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:46.935 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:46 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:46.936 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:46 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:46.936 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:46 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:46.936 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:46 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:46.938 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:46 smithi181 ceph-mon[29765]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:46.939 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:46 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:46.939 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:46 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:46.939 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:46 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:46.939 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:46 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:46.939 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:46 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:46.939 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:46 smithi181 ceph-mon[41183]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:46.939 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:46 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:46.939 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:46 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:46.939 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:46 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:46.939 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:46 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:46.939 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:46 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:47.234 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:47 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:47.137+0000 7f6406c421c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:46:47.234 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:47 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:47.233+0000 7f6406c421c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:46:47.544 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:47 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:47.316+0000 7f6406c421c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:46:47.544 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:47 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:47.465+0000 7f6406c421c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:46:47.544 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:47 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:47.543+0000 7f6406c421c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:46:47.826 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-29T04:46:47.826 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-29T04:46:47.826 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2024-03-29T04:46:47.826 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2024-03-29T04:46:47.826 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0128962 s, 813 MB/s 2024-03-29T04:46:47.826 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2024-03-29T04:46:47.876 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:47 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:47.709+0000 7f6406c421c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:46:48.139 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:47 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:47.885+0000 7f6406c421c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:46:48.139 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:47 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:47.967+0000 7f6406c421c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:46:48.139 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:48.056+0000 7f6406c421c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:46:48.139 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:48.138+0000 7f6406c421c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:46:48.446 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:48.446 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:48.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:48.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:48.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:48.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[29765]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:48.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[29765]: Reconfiguring mgr.y (unknown last config time)... 2024-03-29T04:46:48.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:46:48.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:46:48.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:48.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[29765]: Reconfiguring daemon mgr.y on smithi181 2024-03-29T04:46:48.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:48.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:48.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:48.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:48.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:48.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[41183]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:48.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[41183]: Reconfiguring mgr.y (unknown last config time)... 2024-03-29T04:46:48.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:46:48.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:46:48.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:48.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:48 smithi181 ceph-mon[41183]: Reconfiguring daemon mgr.y on smithi181 2024-03-29T04:46:48.626 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:48 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:48.626 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:48 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:48.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:48 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:48.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:48 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:48.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:48 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:48.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:48 smithi187 ceph-mon[33595]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:48.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:48 smithi187 ceph-mon[33595]: Reconfiguring mgr.y (unknown last config time)... 2024-03-29T04:46:48.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:48 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:46:48.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:48 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:46:48.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:48 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:48.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:48 smithi187 ceph-mon[33595]: Reconfiguring daemon mgr.y on smithi181 2024-03-29T04:46:48.774 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi181:vg_nvme/lv_4 2024-03-29T04:46:49.207 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:46:49.487 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:49 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:49.170+0000 7f6406c421c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:46:49.487 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:49 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:49.300+0000 7f6406c421c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:46:49.487 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:49 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:49.486+0000 7f6406c421c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:46:49.876 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:49 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:49.712+0000 7f6406c421c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:46:50.177 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:49 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:49.880+0000 7f6406c421c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:46:50.177 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:50 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:50.053+0000 7f6406c421c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:46:50.196 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:50 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:50.196 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:50 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:50.196 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:50 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:50.196 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:50 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:50.196 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:50 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:50.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:50 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:50.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:50 smithi181 ceph-mon[41183]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:50.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:50 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:50.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:50 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:50.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:50 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:50.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:50 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:50.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:50 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:50.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:50 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:50.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:50 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:50.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:50 smithi181 ceph-mon[29765]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:50.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:50 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:50.626 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:50 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:50.626 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:50 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:50.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:50 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:50.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:50 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:50.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:50 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:50.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:50 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:50.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:50 smithi187 ceph-mon[33595]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:50.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:50 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:46:50.627 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:50 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:50.303+0000 7f6406c421c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:46:50.627 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:50 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:50.385+0000 7f6406c421c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:46:51.126 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:50 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:50.771+0000 7f6406c421c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:46:51.319 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:51 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:46:51.319 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:51 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:46:51.319 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:51 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:51.320 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:51 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:46:51.320 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:51 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:46:51.320 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:51 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:51.523 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:51 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:46:51.523 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:51 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:46:51.523 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:51 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:51.807 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:51.522+0000 7f6406c421c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:46:51.807 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:51.806+0000 7f6406c421c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:46:52.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:52 smithi187 ceph-mon[33595]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:46:52.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:52 smithi187 ceph-mon[33595]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:52.377 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:52.121+0000 7f6406c421c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:46:52.377 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:46:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:46:52.310+0000 7f6406c421c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:46:52.409 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:52 smithi181 ceph-mon[29765]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:46:52.409 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:52 smithi181 ceph-mon[29765]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:52.409 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:52 smithi181 ceph-mon[41183]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:46:52.409 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:52 smithi181 ceph-mon[41183]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:53.626 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:53 smithi187 ceph-mon[33595]: Standby manager daemon x started 2024-03-29T04:46:53.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:53 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/3543252450' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:46:53.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:53 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/3543252450' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:46:53.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:53 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/3543252450' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:46:53.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:53 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/3543252450' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:46:53.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:53 smithi181 ceph-mon[29765]: Standby manager daemon x started 2024-03-29T04:46:53.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:53 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/3543252450' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:46:53.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:53 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/3543252450' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:46:53.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:53 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/3543252450' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:46:53.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:53 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/3543252450' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:46:53.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:53 smithi181 ceph-mon[41183]: Standby manager daemon x started 2024-03-29T04:46:53.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:53 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/3543252450' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:46:53.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:53 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/3543252450' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:46:53.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:53 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/3543252450' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:46:53.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:53 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/3543252450' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:46:54.626 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:54 smithi187 ceph-mon[33595]: mgrmap e14: y(active, since 64s), standbys: x 2024-03-29T04:46:54.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:54 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:46:54.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:54 smithi187 ceph-mon[33595]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:54.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:54 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2464736572' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "26991441-a6cb-4694-aec3-0f793de6d551"}]: dispatch 2024-03-29T04:46:54.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:54 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2464736572' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "26991441-a6cb-4694-aec3-0f793de6d551"}]': finished 2024-03-29T04:46:54.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:54 smithi187 ceph-mon[33595]: osdmap e5: 1 total, 0 up, 1 in 2024-03-29T04:46:54.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:54 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:46:54.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:54 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3839881803' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:46:54.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:54 smithi181 ceph-mon[29765]: mgrmap e14: y(active, since 64s), standbys: x 2024-03-29T04:46:54.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:54 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:46:54.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:54 smithi181 ceph-mon[29765]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:54.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:54 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2464736572' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "26991441-a6cb-4694-aec3-0f793de6d551"}]: dispatch 2024-03-29T04:46:54.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:54 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2464736572' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "26991441-a6cb-4694-aec3-0f793de6d551"}]': finished 2024-03-29T04:46:54.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:54 smithi181 ceph-mon[29765]: osdmap e5: 1 total, 0 up, 1 in 2024-03-29T04:46:54.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:54 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:46:54.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:54 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3839881803' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:46:54.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:54 smithi181 ceph-mon[41183]: mgrmap e14: y(active, since 64s), standbys: x 2024-03-29T04:46:54.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:54 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:46:54.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:54 smithi181 ceph-mon[41183]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:54.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:54 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2464736572' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "26991441-a6cb-4694-aec3-0f793de6d551"}]: dispatch 2024-03-29T04:46:54.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:54 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2464736572' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "26991441-a6cb-4694-aec3-0f793de6d551"}]': finished 2024-03-29T04:46:54.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:54 smithi181 ceph-mon[41183]: osdmap e5: 1 total, 0 up, 1 in 2024-03-29T04:46:54.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:54 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:46:54.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:54 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3839881803' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:46:56.525 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:56 smithi181 ceph-mon[41183]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:56.526 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:56 smithi181 ceph-mon[29765]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:56.626 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:56 smithi187 ceph-mon[33595]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:58.626 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:46:58 smithi187 ceph-mon[33595]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:58.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:46:58 smithi181 ceph-mon[29765]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:46:58.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:46:58 smithi181 ceph-mon[41183]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:00.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:00 smithi181 ceph-mon[29765]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:00.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:00 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:47:00.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:00 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:00.427 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:00 smithi181 ceph-mon[29765]: Deploying daemon osd.0 on smithi181 2024-03-29T04:47:00.427 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:00 smithi181 ceph-mon[41183]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:00.427 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:00 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:47:00.427 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:00 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:00.427 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:00 smithi181 ceph-mon[41183]: Deploying daemon osd.0 on smithi181 2024-03-29T04:47:00.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:00 smithi187 ceph-mon[33595]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:00.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:00 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:47:00.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:00 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:00.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:00 smithi187 ceph-mon[33595]: Deploying daemon osd.0 on smithi181 2024-03-29T04:47:02.446 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:02 smithi181 ceph-mon[29765]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:02.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:02 smithi181 ceph-mon[41183]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:02.626 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:02 smithi187 ceph-mon[33595]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:04.446 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:04 smithi181 ceph-mon[29765]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:04.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:04.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:04.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:04.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:04 smithi181 ceph-mon[41183]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:04.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:04.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:04.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:04.626 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:04 smithi187 ceph-mon[33595]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:04.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:04.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:04.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:06.446 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:06 smithi181 ceph-mon[29765]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:06.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:06 smithi181 ceph-mon[41183]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:06.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:06 smithi187 ceph-mon[33595]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:07.866 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 0 on host 'smithi181' 2024-03-29T04:47:08.156 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:08.156 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:08.156 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[41183]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:08.156 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[41183]: from='osd.0 [v2:172.21.15.181:6802/135202285,v1:172.21.15.181:6803/135202285]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:47:08.156 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[41183]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:47:08.156 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:08.156 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:08.156 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:08.156 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:08.156 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:08.156 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:08.157 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:08.157 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:08.157 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[29765]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:08.157 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[29765]: from='osd.0 [v2:172.21.15.181:6802/135202285,v1:172.21.15.181:6803/135202285]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:47:08.157 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[29765]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:47:08.157 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:08.157 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:08.157 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:08.157 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:08.157 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:08.157 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:07 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:08.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:07 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:08.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:07 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:08.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:07 smithi187 ceph-mon[33595]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:08.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:07 smithi187 ceph-mon[33595]: from='osd.0 [v2:172.21.15.181:6802/135202285,v1:172.21.15.181:6803/135202285]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:47:08.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:07 smithi187 ceph-mon[33595]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:47:08.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:07 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:08.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:07 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:08.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:07 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:08.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:07 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:08.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:07 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:08.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:07 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:08.473 DEBUG:teuthology.orchestra.run.smithi181:osd.0> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.0.service 2024-03-29T04:47:08.479 INFO:tasks.cephadm:Deploying osd.1 on smithi181 with /dev/vg_nvme/lv_3... 2024-03-29T04:47:08.479 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-29T04:47:09.196 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:08 smithi181 ceph-mon[41183]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:47:09.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:08 smithi181 ceph-mon[41183]: osdmap e6: 1 total, 0 up, 1 in 2024-03-29T04:47:09.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:08 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:47:09.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:08 smithi181 ceph-mon[41183]: from='osd.0 [v2:172.21.15.181:6802/135202285,v1:172.21.15.181:6803/135202285]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:09.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:08 smithi181 ceph-mon[41183]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:09.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:08 smithi181 ceph-mon[29765]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:47:09.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:08 smithi181 ceph-mon[29765]: osdmap e6: 1 total, 0 up, 1 in 2024-03-29T04:47:09.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:08 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:47:09.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:08 smithi181 ceph-mon[29765]: from='osd.0 [v2:172.21.15.181:6802/135202285,v1:172.21.15.181:6803/135202285]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:09.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:08 smithi181 ceph-mon[29765]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:09.198 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:47:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0[45500]: 2024-03-29T04:47:08.933+0000 7f0888321700 -1 osd.0 0 waiting for initial osdmap 2024-03-29T04:47:09.198 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:47:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0[45500]: 2024-03-29T04:47:08.935+0000 7f0881118700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:47:09.212 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:47:09.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:08 smithi187 ceph-mon[33595]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:47:09.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:08 smithi187 ceph-mon[33595]: osdmap e6: 1 total, 0 up, 1 in 2024-03-29T04:47:09.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:08 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:47:09.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:08 smithi187 ceph-mon[33595]: from='osd.0 [v2:172.21.15.181:6802/135202285,v1:172.21.15.181:6803/135202285]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:09.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:08 smithi187 ceph-mon[33595]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:10.009 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:09 smithi181 ceph-mon[41183]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-03-29T04:47:10.009 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:09 smithi181 ceph-mon[41183]: osdmap e7: 1 total, 0 up, 1 in 2024-03-29T04:47:10.009 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:09 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:47:10.010 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:09 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:47:10.010 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:09 smithi181 ceph-mon[41183]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:10.010 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:09 smithi181 ceph-mon[29765]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-03-29T04:47:10.010 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:09 smithi181 ceph-mon[29765]: osdmap e7: 1 total, 0 up, 1 in 2024-03-29T04:47:10.010 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:09 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:47:10.010 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:09 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:47:10.010 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:09 smithi181 ceph-mon[29765]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:10.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:09 smithi187 ceph-mon[33595]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-03-29T04:47:10.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:09 smithi187 ceph-mon[33595]: osdmap e7: 1 total, 0 up, 1 in 2024-03-29T04:47:10.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:09 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:47:10.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:09 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:47:10.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:09 smithi187 ceph-mon[33595]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:47:11.214 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:10 smithi181 ceph-mon[29765]: purged_snaps scrub starts 2024-03-29T04:47:11.214 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:10 smithi181 ceph-mon[29765]: purged_snaps scrub ok 2024-03-29T04:47:11.215 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:10 smithi181 ceph-mon[29765]: osd.0 [v2:172.21.15.181:6802/135202285,v1:172.21.15.181:6803/135202285] boot 2024-03-29T04:47:11.215 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:10 smithi181 ceph-mon[29765]: osdmap e8: 1 total, 1 up, 1 in 2024-03-29T04:47:11.215 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:10 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:47:11.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:10 smithi181 ceph-mon[41183]: purged_snaps scrub starts 2024-03-29T04:47:11.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:10 smithi181 ceph-mon[41183]: purged_snaps scrub ok 2024-03-29T04:47:11.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:10 smithi181 ceph-mon[41183]: osd.0 [v2:172.21.15.181:6802/135202285,v1:172.21.15.181:6803/135202285] boot 2024-03-29T04:47:11.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:10 smithi181 ceph-mon[41183]: osdmap e8: 1 total, 1 up, 1 in 2024-03-29T04:47:11.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:10 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:47:11.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:10 smithi187 ceph-mon[33595]: purged_snaps scrub starts 2024-03-29T04:47:11.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:10 smithi187 ceph-mon[33595]: purged_snaps scrub ok 2024-03-29T04:47:11.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:10 smithi187 ceph-mon[33595]: osd.0 [v2:172.21.15.181:6802/135202285,v1:172.21.15.181:6803/135202285] boot 2024-03-29T04:47:11.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:10 smithi187 ceph-mon[33595]: osdmap e8: 1 total, 1 up, 1 in 2024-03-29T04:47:11.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:10 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:47:12.223 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:11 smithi181 ceph-mon[29765]: osdmap e9: 1 total, 1 up, 1 in 2024-03-29T04:47:12.223 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:11 smithi181 ceph-mon[29765]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:12.224 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:11 smithi181 ceph-mon[41183]: osdmap e9: 1 total, 1 up, 1 in 2024-03-29T04:47:12.224 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:11 smithi181 ceph-mon[41183]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:12.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:11 smithi187 ceph-mon[33595]: osdmap e9: 1 total, 1 up, 1 in 2024-03-29T04:47:12.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:11 smithi187 ceph-mon[33595]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:12.548 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-29T04:47:12.548 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-29T04:47:12.548 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2024-03-29T04:47:12.548 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2024-03-29T04:47:12.548 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0126239 s, 831 MB/s 2024-03-29T04:47:12.548 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2024-03-29T04:47:13.152 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[41183]: Detected new or changed devices on smithi181 2024-03-29T04:47:13.152 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:13.152 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:13.152 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:47:13.152 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[41183]: Adjusting osd_memory_target on smithi181 to 16064M 2024-03-29T04:47:13.152 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:13.152 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:13.152 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:13.152 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:13.152 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[29765]: Detected new or changed devices on smithi181 2024-03-29T04:47:13.153 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:13.153 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:13.153 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:47:13.153 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[29765]: Adjusting osd_memory_target on smithi181 to 16064M 2024-03-29T04:47:13.153 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:13.153 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:13.153 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:13.153 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:13 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:13.176 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi181:vg_nvme/lv_3 2024-03-29T04:47:13.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:13 smithi187 ceph-mon[33595]: Detected new or changed devices on smithi181 2024-03-29T04:47:13.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:13 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:13.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:13 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:13.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:13 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:47:13.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:13 smithi187 ceph-mon[33595]: Adjusting osd_memory_target on smithi181 to 16064M 2024-03-29T04:47:13.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:13 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:13.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:13 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:13.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:13 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:13.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:13 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:13.428 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:47:14.196 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:14 smithi181 ceph-mon[41183]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:14.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:14 smithi181 ceph-mon[29765]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:14.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:14 smithi187 ceph-mon[33595]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:15.603 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:15 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:47:15.603 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:15 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:47:15.603 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:15 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:15.603 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:15 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:47:15.603 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:15 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:47:15.603 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:15 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:15.626 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:15 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:47:15.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:15 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:47:15.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:15 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:16.609 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:16 smithi181 ceph-mon[41183]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:47:16.609 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:16 smithi181 ceph-mon[41183]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:16.609 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:16 smithi181 ceph-mon[29765]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:47:16.609 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:16 smithi181 ceph-mon[29765]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:16.626 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:16 smithi187 ceph-mon[33595]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:47:16.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:16 smithi187 ceph-mon[33595]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:18.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:18 smithi187 ceph-mon[33595]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:18.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:18 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/4226487658' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04b645ce-4eb8-43bc-9f28-e3be932f4b1e"}]: dispatch 2024-03-29T04:47:18.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:18 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/4226487658' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04b645ce-4eb8-43bc-9f28-e3be932f4b1e"}]': finished 2024-03-29T04:47:18.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:18 smithi187 ceph-mon[33595]: osdmap e10: 2 total, 1 up, 2 in 2024-03-29T04:47:18.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:18 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:18.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:18 smithi181 ceph-mon[29765]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:18.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:18 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/4226487658' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04b645ce-4eb8-43bc-9f28-e3be932f4b1e"}]: dispatch 2024-03-29T04:47:18.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:18 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/4226487658' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04b645ce-4eb8-43bc-9f28-e3be932f4b1e"}]': finished 2024-03-29T04:47:18.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:18 smithi181 ceph-mon[29765]: osdmap e10: 2 total, 1 up, 2 in 2024-03-29T04:47:18.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:18 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:18.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:18 smithi181 ceph-mon[41183]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:18.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:18 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/4226487658' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04b645ce-4eb8-43bc-9f28-e3be932f4b1e"}]: dispatch 2024-03-29T04:47:18.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:18 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/4226487658' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04b645ce-4eb8-43bc-9f28-e3be932f4b1e"}]': finished 2024-03-29T04:47:18.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:18 smithi181 ceph-mon[41183]: osdmap e10: 2 total, 1 up, 2 in 2024-03-29T04:47:18.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:18 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:19.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:19 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2122108568' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:47:19.696 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:19 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2122108568' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:47:19.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:19 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2122108568' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:47:20.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:20 smithi181 ceph-mon[29765]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:20.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:20 smithi181 ceph-mon[41183]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:20.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:20 smithi187 ceph-mon[33595]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:22.491 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:22 smithi181 ceph-mon[29765]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:22.492 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:22 smithi181 ceph-mon[41183]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:22.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:22 smithi187 ceph-mon[33595]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:24.491 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:24 smithi181 ceph-mon[29765]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:24.491 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:24 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:47:24.491 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:24 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:24.491 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:24 smithi181 ceph-mon[41183]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:24.491 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:24 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:47:24.491 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:24 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:24.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:24 smithi187 ceph-mon[33595]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:24.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:24 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:47:24.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:24 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:25.536 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:25 smithi181 ceph-mon[29765]: Deploying daemon osd.1 on smithi181 2024-03-29T04:47:25.536 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:25 smithi181 ceph-mon[41183]: Deploying daemon osd.1 on smithi181 2024-03-29T04:47:25.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:25 smithi187 ceph-mon[33595]: Deploying daemon osd.1 on smithi181 2024-03-29T04:47:26.442 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:26 smithi181 ceph-mon[29765]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:26.442 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:26 smithi181 ceph-mon[41183]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:26.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:26 smithi187 ceph-mon[33595]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:28.508 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:28 smithi181 ceph-mon[29765]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:28.508 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:28 smithi181 ceph-mon[41183]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:28.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:28 smithi187 ceph-mon[33595]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:30.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:30 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:30.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:30 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:30.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:30 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:30.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:30 smithi187 ceph-mon[33595]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:30.408 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:30 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:30.409 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:30 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:30.409 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:30 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:30.409 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:30 smithi181 ceph-mon[29765]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:30.409 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:30.409 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:30.409 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:30.409 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:30 smithi181 ceph-mon[41183]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:32.201 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 1 on host 'smithi181' 2024-03-29T04:47:32.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:32 smithi181 ceph-mon[29765]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:32.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:32 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:32.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:32 smithi181 ceph-mon[29765]: from='osd.1 [v2:172.21.15.181:6810/3021678652,v1:172.21.15.181:6811/3021678652]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:47:32.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:32 smithi181 ceph-mon[29765]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:47:32.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:32 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:32.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:32 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:32.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:32 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:32.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:32 smithi181 ceph-mon[41183]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:32.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:32 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:32.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:32 smithi181 ceph-mon[41183]: from='osd.1 [v2:172.21.15.181:6810/3021678652,v1:172.21.15.181:6811/3021678652]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:47:32.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:32 smithi181 ceph-mon[41183]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:47:32.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:32 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:32.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:32 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:32.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:32 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:32.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:32 smithi187 ceph-mon[33595]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:32.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:32 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:32.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:32 smithi187 ceph-mon[33595]: from='osd.1 [v2:172.21.15.181:6810/3021678652,v1:172.21.15.181:6811/3021678652]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:47:32.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:32 smithi187 ceph-mon[33595]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:47:32.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:32 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:32.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:32 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:32.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:32 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:32.822 DEBUG:teuthology.orchestra.run.smithi181:osd.1> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.1.service 2024-03-29T04:47:32.824 INFO:tasks.cephadm:Deploying osd.2 on smithi181 with /dev/vg_nvme/lv_2... 2024-03-29T04:47:32.824 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-29T04:47:33.643 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:47:34.427 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:34 smithi181 ceph-mon[41183]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T04:47:34.427 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:34 smithi181 ceph-mon[41183]: osdmap e11: 2 total, 1 up, 2 in 2024-03-29T04:47:34.427 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:34 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:34.428 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:34 smithi181 ceph-mon[41183]: from='osd.1 [v2:172.21.15.181:6810/3021678652,v1:172.21.15.181:6811/3021678652]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:34.428 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:34 smithi181 ceph-mon[41183]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:34.428 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:34 smithi181 ceph-mon[41183]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:34.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:34 smithi181 ceph-mon[29765]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T04:47:34.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:34 smithi181 ceph-mon[29765]: osdmap e11: 2 total, 1 up, 2 in 2024-03-29T04:47:34.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:34 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:34.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:34 smithi181 ceph-mon[29765]: from='osd.1 [v2:172.21.15.181:6810/3021678652,v1:172.21.15.181:6811/3021678652]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:34.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:34 smithi181 ceph-mon[29765]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:34.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:34 smithi181 ceph-mon[29765]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:34.428 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 04:47:34 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1[51865]: 2024-03-29T04:47:34.159+0000 7f9092dcf700 -1 osd.1 0 waiting for initial osdmap 2024-03-29T04:47:34.429 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 04:47:34 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1[51865]: 2024-03-29T04:47:34.162+0000 7f908fbce700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:47:34.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:34 smithi187 ceph-mon[33595]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T04:47:34.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:34 smithi187 ceph-mon[33595]: osdmap e11: 2 total, 1 up, 2 in 2024-03-29T04:47:34.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:34 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:34.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:34 smithi187 ceph-mon[33595]: from='osd.1 [v2:172.21.15.181:6810/3021678652,v1:172.21.15.181:6811/3021678652]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:34.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:34 smithi187 ceph-mon[33595]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:34.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:34 smithi187 ceph-mon[33595]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:47:35.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:35 smithi181 ceph-mon[29765]: purged_snaps scrub starts 2024-03-29T04:47:35.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:35 smithi181 ceph-mon[29765]: purged_snaps scrub ok 2024-03-29T04:47:35.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:35 smithi181 ceph-mon[29765]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-03-29T04:47:35.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:35 smithi181 ceph-mon[29765]: osdmap e12: 2 total, 1 up, 2 in 2024-03-29T04:47:35.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:35 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:35.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:35 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:35.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:35 smithi181 ceph-mon[41183]: purged_snaps scrub starts 2024-03-29T04:47:35.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:35 smithi181 ceph-mon[41183]: purged_snaps scrub ok 2024-03-29T04:47:35.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:35 smithi181 ceph-mon[41183]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-03-29T04:47:35.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:35 smithi181 ceph-mon[41183]: osdmap e12: 2 total, 1 up, 2 in 2024-03-29T04:47:35.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:35 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:35.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:35 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:35.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:35 smithi187 ceph-mon[33595]: purged_snaps scrub starts 2024-03-29T04:47:35.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:35 smithi187 ceph-mon[33595]: purged_snaps scrub ok 2024-03-29T04:47:35.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:35 smithi187 ceph-mon[33595]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-03-29T04:47:35.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:35 smithi187 ceph-mon[33595]: osdmap e12: 2 total, 1 up, 2 in 2024-03-29T04:47:35.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:35 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:35.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:35 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:36.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:36 smithi181 ceph-mon[29765]: osd.1 [v2:172.21.15.181:6810/3021678652,v1:172.21.15.181:6811/3021678652] boot 2024-03-29T04:47:36.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:36 smithi181 ceph-mon[29765]: osdmap e13: 2 total, 2 up, 2 in 2024-03-29T04:47:36.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:36 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:36.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:36 smithi181 ceph-mon[29765]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:36.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:36 smithi181 ceph-mon[41183]: osd.1 [v2:172.21.15.181:6810/3021678652,v1:172.21.15.181:6811/3021678652] boot 2024-03-29T04:47:36.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:36 smithi181 ceph-mon[41183]: osdmap e13: 2 total, 2 up, 2 in 2024-03-29T04:47:36.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:36 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:36.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:36 smithi181 ceph-mon[41183]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:36.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:36 smithi187 ceph-mon[33595]: osd.1 [v2:172.21.15.181:6810/3021678652,v1:172.21.15.181:6811/3021678652] boot 2024-03-29T04:47:36.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:36 smithi187 ceph-mon[33595]: osdmap e13: 2 total, 2 up, 2 in 2024-03-29T04:47:36.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:36 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:47:36.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:36 smithi187 ceph-mon[33595]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:36.687 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-29T04:47:36.687 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-29T04:47:36.687 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2024-03-29T04:47:36.687 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2024-03-29T04:47:36.687 INFO:teuthology.orchestra.run.smithi181.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0147147 s, 713 MB/s 2024-03-29T04:47:36.687 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2024-03-29T04:47:37.383 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi181:vg_nvme/lv_2 2024-03-29T04:47:37.632 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:47:38.011 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[41183]: Detected new or changed devices on smithi181 2024-03-29T04:47:38.012 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:38.012 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:38.012 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:47:38.012 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:47:38.012 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[41183]: Adjusting osd_memory_target on smithi181 to 8032M 2024-03-29T04:47:38.012 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:38.012 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:38.012 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:38.012 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:38.012 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:38.012 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:38.012 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:38.013 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:38.013 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[41183]: osdmap e14: 2 total, 2 up, 2 in 2024-03-29T04:47:38.013 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[41183]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:38.013 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[29765]: Detected new or changed devices on smithi181 2024-03-29T04:47:38.013 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:38.013 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:38.013 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:47:38.013 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:47:38.013 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[29765]: Adjusting osd_memory_target on smithi181 to 8032M 2024-03-29T04:47:38.013 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:38.013 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:38.013 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:38.014 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:38.014 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:38.014 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:38.014 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:38.014 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:38.014 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[29765]: osdmap e14: 2 total, 2 up, 2 in 2024-03-29T04:47:38.014 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:37 smithi181 ceph-mon[29765]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:38.127 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:37 smithi187 ceph-mon[33595]: Detected new or changed devices on smithi181 2024-03-29T04:47:38.127 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:38.127 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:38.127 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:47:38.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:47:38.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:37 smithi187 ceph-mon[33595]: Adjusting osd_memory_target on smithi181 to 8032M 2024-03-29T04:47:38.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:38.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:38.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:38.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:38.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:38.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:38.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:38.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:38.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:37 smithi187 ceph-mon[33595]: osdmap e14: 2 total, 2 up, 2 in 2024-03-29T04:47:38.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:37 smithi187 ceph-mon[33595]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:40.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:40 smithi181 ceph-mon[29765]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:40.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:40 smithi181 ceph-mon[29765]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:47:40.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:47:40.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:47:40.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:40.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:40 smithi181 ceph-mon[41183]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:40.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:40 smithi181 ceph-mon[41183]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:47:40.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:47:40.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:47:40.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:40.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:40 smithi187 ceph-mon[33595]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:40.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:40 smithi187 ceph-mon[33595]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:47:40.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:47:40.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:47:40.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:42.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:42 smithi187 ceph-mon[33595]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:42.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:42 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3631758536' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f72d641c-5d86-4d3a-bf90-d99c1c46a805"}]: dispatch 2024-03-29T04:47:42.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:42 smithi187 ceph-mon[33595]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f72d641c-5d86-4d3a-bf90-d99c1c46a805"}]: dispatch 2024-03-29T04:47:42.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:42 smithi187 ceph-mon[33595]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f72d641c-5d86-4d3a-bf90-d99c1c46a805"}]': finished 2024-03-29T04:47:42.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:42 smithi187 ceph-mon[33595]: osdmap e15: 3 total, 2 up, 3 in 2024-03-29T04:47:42.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:42 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:47:42.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:42 smithi181 ceph-mon[29765]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:42.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:42 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3631758536' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f72d641c-5d86-4d3a-bf90-d99c1c46a805"}]: dispatch 2024-03-29T04:47:42.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:42 smithi181 ceph-mon[29765]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f72d641c-5d86-4d3a-bf90-d99c1c46a805"}]: dispatch 2024-03-29T04:47:42.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:42 smithi181 ceph-mon[29765]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f72d641c-5d86-4d3a-bf90-d99c1c46a805"}]': finished 2024-03-29T04:47:42.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:42 smithi181 ceph-mon[29765]: osdmap e15: 3 total, 2 up, 3 in 2024-03-29T04:47:42.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:42 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:47:42.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:42 smithi181 ceph-mon[41183]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:42.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:42 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3631758536' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f72d641c-5d86-4d3a-bf90-d99c1c46a805"}]: dispatch 2024-03-29T04:47:42.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:42 smithi181 ceph-mon[41183]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f72d641c-5d86-4d3a-bf90-d99c1c46a805"}]: dispatch 2024-03-29T04:47:42.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:42 smithi181 ceph-mon[41183]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f72d641c-5d86-4d3a-bf90-d99c1c46a805"}]': finished 2024-03-29T04:47:42.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:42 smithi181 ceph-mon[41183]: osdmap e15: 3 total, 2 up, 3 in 2024-03-29T04:47:42.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:42 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:47:43.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:43 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2527510490' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:47:43.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:43 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2527510490' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:47:43.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:43 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2527510490' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:47:44.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:44 smithi187 ceph-mon[33595]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:44.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:44 smithi181 ceph-mon[29765]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:44.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:44 smithi181 ceph-mon[41183]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:46.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:46 smithi187 ceph-mon[33595]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:46.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:46 smithi181 ceph-mon[41183]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:46.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:46 smithi181 ceph-mon[29765]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:48.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:48 smithi181 ceph-mon[29765]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:48.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:48 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:47:48.447 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:48 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:48.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:48 smithi181 ceph-mon[41183]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:48.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:48 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:47:48.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:48 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:48.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:48 smithi187 ceph-mon[33595]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:48.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:48 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:47:48.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:48 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:49.619 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:49 smithi181 ceph-mon[41183]: Deploying daemon osd.2 on smithi181 2024-03-29T04:47:49.619 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:49 smithi181 ceph-mon[29765]: Deploying daemon osd.2 on smithi181 2024-03-29T04:47:49.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:49 smithi187 ceph-mon[33595]: Deploying daemon osd.2 on smithi181 2024-03-29T04:47:50.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:50 smithi187 ceph-mon[33595]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:50.650 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:50 smithi181 ceph-mon[29765]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:50.650 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:50 smithi181 ceph-mon[41183]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:52.607 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:52 smithi181 ceph-mon[41183]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:52.608 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:52 smithi181 ceph-mon[29765]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:52.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:52 smithi187 ceph-mon[33595]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:54.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:54 smithi181 ceph-mon[29765]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:54.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:54 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:54.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:54 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:54.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:54 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:54.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:54 smithi181 ceph-mon[41183]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:54.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:54 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:54.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:54 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:54.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:54 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:54.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:54 smithi187 ceph-mon[33595]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:54.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:54 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:54.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:54 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:54.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:54 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:56.652 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:56 smithi181 ceph-mon[29765]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:56.652 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:56 smithi181 ceph-mon[41183]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:56.731 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 2 on host 'smithi181' 2024-03-29T04:47:56.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:56 smithi187 ceph-mon[33595]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:57.321 DEBUG:teuthology.orchestra.run.smithi181:osd.2> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.2.service 2024-03-29T04:47:57.323 INFO:tasks.cephadm:Deploying osd.3 on smithi181 with /dev/vg_nvme/lv_1... 2024-03-29T04:47:57.323 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-29T04:47:57.637 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:47:57.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:57 smithi181 ceph-mon[29765]: from='osd.2 [v2:172.21.15.181:6818/352673207,v1:172.21.15.181:6819/352673207]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:47:57.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:57 smithi181 ceph-mon[29765]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:47:57.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:57 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:57.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:57 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:57.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:57 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:57.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:57 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:57.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:57 smithi181 ceph-mon[41183]: from='osd.2 [v2:172.21.15.181:6818/352673207,v1:172.21.15.181:6819/352673207]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:47:57.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:57 smithi181 ceph-mon[41183]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:47:57.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:57 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:57.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:57 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:57.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:57 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:57.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:57 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:57.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:57 smithi187 ceph-mon[33595]: from='osd.2 [v2:172.21.15.181:6818/352673207,v1:172.21.15.181:6819/352673207]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:47:57.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:57 smithi187 ceph-mon[33595]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:47:57.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:57 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:57.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:57 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:57.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:57 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:57.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:57 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:47:58.713 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 04:47:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2[57869]: 2024-03-29T04:47:58.459+0000 7f682c5cd700 -1 osd.2 0 waiting for initial osdmap 2024-03-29T04:47:58.714 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 04:47:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2[57869]: 2024-03-29T04:47:58.462+0000 7f6825bc5700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:47:58.714 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:58 smithi181 ceph-mon[29765]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:58.714 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:58 smithi181 ceph-mon[29765]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T04:47:58.714 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:58 smithi181 ceph-mon[29765]: osdmap e16: 3 total, 2 up, 3 in 2024-03-29T04:47:58.714 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:58 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:47:58.714 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:58 smithi181 ceph-mon[29765]: from='osd.2 [v2:172.21.15.181:6818/352673207,v1:172.21.15.181:6819/352673207]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:58.714 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:58 smithi181 ceph-mon[29765]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:58.715 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:58 smithi181 ceph-mon[41183]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:58.715 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:58 smithi181 ceph-mon[41183]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T04:47:58.715 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:58 smithi181 ceph-mon[41183]: osdmap e16: 3 total, 2 up, 3 in 2024-03-29T04:47:58.715 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:58 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:47:58.715 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:58 smithi181 ceph-mon[41183]: from='osd.2 [v2:172.21.15.181:6818/352673207,v1:172.21.15.181:6819/352673207]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:58.715 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:58 smithi181 ceph-mon[41183]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:58.779 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:58 smithi187 ceph-mon[33595]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:47:58.779 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:58 smithi187 ceph-mon[33595]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T04:47:58.779 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:58 smithi187 ceph-mon[33595]: osdmap e16: 3 total, 2 up, 3 in 2024-03-29T04:47:58.779 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:58 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:47:58.779 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:58 smithi187 ceph-mon[33595]: from='osd.2 [v2:172.21.15.181:6818/352673207,v1:172.21.15.181:6819/352673207]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:58.780 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:58 smithi187 ceph-mon[33595]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:47:59.660 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:59 smithi181 ceph-mon[29765]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-03-29T04:47:59.661 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:59 smithi181 ceph-mon[29765]: osdmap e17: 3 total, 2 up, 3 in 2024-03-29T04:47:59.661 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:59 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:47:59.661 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:47:59 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:47:59.661 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:59 smithi181 ceph-mon[41183]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-03-29T04:47:59.661 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:59 smithi181 ceph-mon[41183]: osdmap e17: 3 total, 2 up, 3 in 2024-03-29T04:47:59.661 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:59 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:47:59.661 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:47:59 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:47:59.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:59 smithi187 ceph-mon[33595]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-03-29T04:47:59.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:59 smithi187 ceph-mon[33595]: osdmap e17: 3 total, 2 up, 3 in 2024-03-29T04:47:59.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:59 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:47:59.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:47:59 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:48:00.116 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-29T04:48:00.116 INFO:teuthology.orchestra.run.smithi181.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-29T04:48:00.116 INFO:teuthology.orchestra.run.smithi181.stderr: stderr: 10+0 records in 2024-03-29T04:48:00.116 INFO:teuthology.orchestra.run.smithi181.stderr:10+0 records out 2024-03-29T04:48:00.116 INFO:teuthology.orchestra.run.smithi181.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0124752 s, 841 MB/s 2024-03-29T04:48:00.116 INFO:teuthology.orchestra.run.smithi181.stderr:--> Zapping successful for: 2024-03-29T04:48:00.794 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:00 smithi181 ceph-mon[41183]: purged_snaps scrub starts 2024-03-29T04:48:00.794 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:00 smithi181 ceph-mon[41183]: purged_snaps scrub ok 2024-03-29T04:48:00.794 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:00 smithi181 ceph-mon[41183]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:48:00.794 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:00 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:48:00.794 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:00 smithi181 ceph-mon[41183]: osd.2 [v2:172.21.15.181:6818/352673207,v1:172.21.15.181:6819/352673207] boot 2024-03-29T04:48:00.797 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:00 smithi181 ceph-mon[41183]: osdmap e18: 3 total, 3 up, 3 in 2024-03-29T04:48:00.797 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:00 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:48:00.797 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:00 smithi181 ceph-mon[29765]: purged_snaps scrub starts 2024-03-29T04:48:00.797 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:00 smithi181 ceph-mon[29765]: purged_snaps scrub ok 2024-03-29T04:48:00.797 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:00 smithi181 ceph-mon[29765]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:48:00.798 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:00 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:48:00.798 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:00 smithi181 ceph-mon[29765]: osd.2 [v2:172.21.15.181:6818/352673207,v1:172.21.15.181:6819/352673207] boot 2024-03-29T04:48:00.798 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:00 smithi181 ceph-mon[29765]: osdmap e18: 3 total, 3 up, 3 in 2024-03-29T04:48:00.798 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:00 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:48:00.819 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi181:vg_nvme/lv_1 2024-03-29T04:48:00.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:00 smithi187 ceph-mon[33595]: purged_snaps scrub starts 2024-03-29T04:48:00.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:00 smithi187 ceph-mon[33595]: purged_snaps scrub ok 2024-03-29T04:48:00.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:00 smithi187 ceph-mon[33595]: pgmap v72: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:48:00.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:00 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:48:00.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:00 smithi187 ceph-mon[33595]: osd.2 [v2:172.21.15.181:6818/352673207,v1:172.21.15.181:6819/352673207] boot 2024-03-29T04:48:00.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:00 smithi187 ceph-mon[33595]: osdmap e18: 3 total, 3 up, 3 in 2024-03-29T04:48:00.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:00 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:48:01.385 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:48:01.834 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:01 smithi181 ceph-mon[29765]: osdmap e19: 3 total, 3 up, 3 in 2024-03-29T04:48:01.835 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:01 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-29T04:48:01.835 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:01 smithi181 ceph-mon[41183]: osdmap e19: 3 total, 3 up, 3 in 2024-03-29T04:48:01.835 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:01 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-29T04:48:01.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:01 smithi187 ceph-mon[33595]: osdmap e19: 3 total, 3 up, 3 in 2024-03-29T04:48:01.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:01 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-29T04:48:02.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:02 smithi181 ceph-mon[29765]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:02.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:02 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-29T04:48:02.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:02 smithi181 ceph-mon[29765]: osdmap e20: 3 total, 3 up, 3 in 2024-03-29T04:48:02.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:02 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:48:02.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:02 smithi181 ceph-mon[41183]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:02.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:02 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-29T04:48:02.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:02 smithi181 ceph-mon[41183]: osdmap e20: 3 total, 3 up, 3 in 2024-03-29T04:48:02.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:02 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:48:02.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:02 smithi187 ceph-mon[33595]: pgmap v75: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:02.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:02 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-29T04:48:02.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:02 smithi187 ceph-mon[33595]: osdmap e20: 3 total, 3 up, 3 in 2024-03-29T04:48:02.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:02 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:48:03.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:48:03.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: osdmap e21: 3 total, 3 up, 3 in 2024-03-29T04:48:03.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: Detected new or changed devices on smithi181 2024-03-29T04:48:03.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:03.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:03.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:03.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:03.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:03.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: Adjusting osd_memory_target on smithi181 to 5354M 2024-03-29T04:48:03.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:03.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:03.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:03.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:03.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:03.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:03.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:03.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:03 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:03.913 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:48:03 smithi181 sudo[62273]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:48:03.913 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:48:03 smithi181 sudo[62273]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:48:03.913 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:48:03 smithi181 sudo[62273]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:48:03.913 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:48:03 smithi181 sudo[62273]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:48:03.914 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:48:03.914 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: osdmap e21: 3 total, 3 up, 3 in 2024-03-29T04:48:03.914 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: Detected new or changed devices on smithi181 2024-03-29T04:48:03.914 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:03.914 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:03.914 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:03.914 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:03.914 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:03.914 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: Adjusting osd_memory_target on smithi181 to 5354M 2024-03-29T04:48:03.914 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:03.915 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:03.915 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:03.915 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:03.915 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:03.915 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:03.915 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:03.915 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:03.915 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:48:03.915 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: osdmap e21: 3 total, 3 up, 3 in 2024-03-29T04:48:03.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: Detected new or changed devices on smithi181 2024-03-29T04:48:03.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:03.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:03.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:03.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:03.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:03.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: Adjusting osd_memory_target on smithi181 to 5354M 2024-03-29T04:48:03.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:03.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:03.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:03.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:03.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:03.917 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:03.917 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:03.917 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:03 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:04.192 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:48:03 smithi181 sudo[62308]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:48:04.192 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:48:03 smithi181 sudo[62308]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:48:04.192 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:48:03 smithi181 sudo[62308]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:48:04.192 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:48:03 smithi181 sudo[62308]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:48:04.444 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 04:48:04 smithi181 sudo[62407]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:48:04.444 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 04:48:04 smithi181 sudo[62407]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:48:04.444 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 04:48:04 smithi181 sudo[62407]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:48:04.444 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 04:48:04 smithi181 sudo[62407]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:48:04.697 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 04:48:04 smithi181 sudo[62429]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:48:04.697 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 04:48:04 smithi181 sudo[62429]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:48:04.697 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 04:48:04 smithi181 sudo[62429]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:48:04.697 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 04:48:04 smithi181 sudo[62429]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:48:04.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:04 smithi181 ceph-mon[41183]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:04.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:48:04.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:48:04.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:04.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:04 smithi181 ceph-mon[41183]: osdmap e22: 3 total, 3 up, 3 in 2024-03-29T04:48:04.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:04.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:04 smithi181 ceph-mon[29765]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:04.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:48:04.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:48:04.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:04.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:04 smithi181 ceph-mon[29765]: osdmap e22: 3 total, 3 up, 3 in 2024-03-29T04:48:04.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:04.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:04 smithi187 ceph-mon[33595]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:04.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:48:04.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:48:04.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:04.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:04 smithi187 ceph-mon[33595]: osdmap e22: 3 total, 3 up, 3 in 2024-03-29T04:48:04.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:04.964 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 04:48:04 smithi181 sudo[62466]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:48:04.964 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 04:48:04 smithi181 sudo[62466]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:48:04.965 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 04:48:04 smithi181 sudo[62466]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:48:04.965 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 04:48:04 smithi181 sudo[62466]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:48:05.222 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 04:48:04 smithi181 sudo[62469]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:48:05.223 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 04:48:04 smithi181 sudo[62469]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:48:05.223 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 04:48:04 smithi181 sudo[62469]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:48:05.223 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 04:48:04 smithi181 sudo[62469]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:48:05.223 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 sudo[62481]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:48:05.223 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 sudo[62481]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:48:05.223 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 sudo[62481]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:48:05.223 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 sudo[62481]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:48:05.507 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 sudo[62505]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:48:05.507 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 sudo[62505]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:48:05.507 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 sudo[62505]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:48:05.507 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 sudo[62505]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:48:05.961 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 sudo[36420]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:48:05.961 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 sudo[36420]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:48:05.962 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 sudo[36420]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:48:05.962 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 sudo[36420]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:48:05.962 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 sudo[36423]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:48:05.962 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 sudo[36423]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:48:05.962 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 sudo[36423]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:48:05.962 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 sudo[36423]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:48:06.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[29765]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:06.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[29765]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:48:06.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[29765]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:48:06.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:48:06.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:48:06.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:48:06.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:48:06.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:48:06.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:48:06.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[41183]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:06.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[41183]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:48:06.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[41183]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:48:06.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:48:06.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:48:06.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:48:06.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:48:06.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:48:06.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:05 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:48:06.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 ceph-mon[33595]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi181:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:06.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 ceph-mon[33595]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:48:06.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 ceph-mon[33595]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:48:06.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:48:06.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:48:06.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:48:06.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:48:06.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:48:06.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:05 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:48:06.500 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 sudo[62559]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:48:06.500 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 sudo[62559]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:48:06.500 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 sudo[62559]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:48:06.500 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 sudo[62559]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:48:06.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 sudo[62582]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:48:06.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 sudo[62582]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:48:06.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 sudo[62582]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:48:06.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 sudo[62582]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:48:07.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:06 smithi187 ceph-mon[33595]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:07.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:06 smithi187 ceph-mon[33595]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:48:07.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:06 smithi187 ceph-mon[33595]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:48:07.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:06 smithi187 ceph-mon[33595]: mgrmap e15: y(active, since 2m), standbys: x 2024-03-29T04:48:07.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:06 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:07.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:06 smithi187 ceph-mon[33595]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:48:07.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:06 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:48:07.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:06 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:48:07.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:06 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:48:07.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:06 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3487818876' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63ebbc1a-43b5-4990-87aa-f8f0ff975830"}]: dispatch 2024-03-29T04:48:07.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:06 smithi187 ceph-mon[33595]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:48:07.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:06 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3487818876' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "63ebbc1a-43b5-4990-87aa-f8f0ff975830"}]': finished 2024-03-29T04:48:07.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:06 smithi187 ceph-mon[33595]: osdmap e23: 4 total, 3 up, 4 in 2024-03-29T04:48:07.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:06 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:48:07.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[41183]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:07.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[41183]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:48:07.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[41183]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:48:07.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[41183]: mgrmap e15: y(active, since 2m), standbys: x 2024-03-29T04:48:07.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:07.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[41183]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:48:07.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:48:07.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:48:07.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:48:07.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3487818876' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63ebbc1a-43b5-4990-87aa-f8f0ff975830"}]: dispatch 2024-03-29T04:48:07.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[41183]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:48:07.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3487818876' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "63ebbc1a-43b5-4990-87aa-f8f0ff975830"}]': finished 2024-03-29T04:48:07.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[41183]: osdmap e23: 4 total, 3 up, 4 in 2024-03-29T04:48:07.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:48:07.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[29765]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:07.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[29765]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:48:07.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[29765]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:48:07.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[29765]: mgrmap e15: y(active, since 2m), standbys: x 2024-03-29T04:48:07.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:07.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[29765]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:48:07.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:48:07.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:48:07.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:48:07.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3487818876' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "63ebbc1a-43b5-4990-87aa-f8f0ff975830"}]: dispatch 2024-03-29T04:48:07.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[29765]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:48:07.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3487818876' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "63ebbc1a-43b5-4990-87aa-f8f0ff975830"}]': finished 2024-03-29T04:48:07.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[29765]: osdmap e23: 4 total, 3 up, 4 in 2024-03-29T04:48:07.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:06 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:48:08.350 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:07 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/895074641' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:48:08.350 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:07 smithi181 ceph-mon[29765]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:08.350 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:07 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/895074641' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:48:08.350 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:07 smithi181 ceph-mon[41183]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:08.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:07 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/895074641' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:48:08.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:07 smithi187 ceph-mon[33595]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:10.580 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:10 smithi181 ceph-mon[41183]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:10.580 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:10 smithi181 ceph-mon[29765]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:10.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:10 smithi187 ceph-mon[33595]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:12.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:12 smithi187 ceph-mon[33595]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-03-29T04:48:12.629 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:12 smithi181 ceph-mon[29765]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-03-29T04:48:12.629 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:12 smithi181 ceph-mon[41183]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-03-29T04:48:13.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:13 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:48:13.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:13 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:13.650 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:13 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:48:13.650 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:13 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:13.650 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:13 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:48:13.650 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:13 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:14.369 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:14 smithi181 ceph-mon[29765]: Deploying daemon osd.3 on smithi181 2024-03-29T04:48:14.370 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:14 smithi181 ceph-mon[29765]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-03-29T04:48:14.370 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:14 smithi181 ceph-mon[41183]: Deploying daemon osd.3 on smithi181 2024-03-29T04:48:14.370 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:14 smithi181 ceph-mon[41183]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-03-29T04:48:14.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:14 smithi187 ceph-mon[33595]: Deploying daemon osd.3 on smithi181 2024-03-29T04:48:14.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:14 smithi187 ceph-mon[33595]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-03-29T04:48:16.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:16 smithi187 ceph-mon[33595]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-03-29T04:48:16.649 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:16 smithi181 ceph-mon[29765]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-03-29T04:48:16.649 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:16 smithi181 ceph-mon[41183]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-03-29T04:48:18.591 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:18 smithi181 ceph-mon[29765]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.7 KiB/s rd, 73 KiB/s wr, 5 op/s 2024-03-29T04:48:18.591 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:18 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:18.591 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:18 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:18.591 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:18 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:18.591 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:18 smithi181 ceph-mon[41183]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.7 KiB/s rd, 73 KiB/s wr, 5 op/s 2024-03-29T04:48:18.591 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:18 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:18.591 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:18 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:18.591 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:18 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:18.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:18 smithi187 ceph-mon[33595]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.7 KiB/s rd, 73 KiB/s wr, 5 op/s 2024-03-29T04:48:18.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:18 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:18.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:18 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:18.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:18 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:20.447 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:20 smithi181 ceph-mon[41183]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-29T04:48:20.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:20 smithi181 ceph-mon[29765]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-29T04:48:20.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:20 smithi187 ceph-mon[33595]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-29T04:48:21.643 INFO:teuthology.orchestra.run.smithi181.stdout:Created osd(s) 3 on host 'smithi181' 2024-03-29T04:48:22.244 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:21 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:22.245 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:21 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:22.245 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:21 smithi181 ceph-mon[29765]: from='osd.3 [v2:172.21.15.181:6826/1785806254,v1:172.21.15.181:6827/1785806254]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:48:22.245 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:21 smithi181 ceph-mon[29765]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:48:22.245 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:21 smithi181 ceph-mon[29765]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-29T04:48:22.245 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:21 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:22.245 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:21 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:22.245 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:21 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:22.245 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:21 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:22.245 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:21 smithi181 ceph-mon[41183]: from='osd.3 [v2:172.21.15.181:6826/1785806254,v1:172.21.15.181:6827/1785806254]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:48:22.246 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:21 smithi181 ceph-mon[41183]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:48:22.246 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:21 smithi181 ceph-mon[41183]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-29T04:48:22.246 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:21 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:22.246 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:21 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:22.271 DEBUG:teuthology.orchestra.run.smithi181:osd.3> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.3.service 2024-03-29T04:48:22.273 INFO:tasks.cephadm:Deploying osd.4 on smithi187 with /dev/vg_nvme/lv_4... 2024-03-29T04:48:22.274 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-29T04:48:22.304 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:21 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:22.304 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:21 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:22.304 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:21 smithi187 ceph-mon[33595]: from='osd.3 [v2:172.21.15.181:6826/1785806254,v1:172.21.15.181:6827/1785806254]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:48:22.304 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:21 smithi187 ceph-mon[33595]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:48:22.304 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:21 smithi187 ceph-mon[33595]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-29T04:48:22.304 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:21 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:22.304 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:21 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:22.529 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:48:23.250 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:22 smithi187 ceph-mon[33595]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T04:48:23.250 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:22 smithi187 ceph-mon[33595]: osdmap e24: 4 total, 3 up, 4 in 2024-03-29T04:48:23.250 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:22 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:48:23.251 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:22 smithi187 ceph-mon[33595]: from='osd.3 [v2:172.21.15.181:6826/1785806254,v1:172.21.15.181:6827/1785806254]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:48:23.251 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:22 smithi187 ceph-mon[33595]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:48:23.368 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:22 smithi181 ceph-mon[41183]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T04:48:23.368 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:22 smithi181 ceph-mon[41183]: osdmap e24: 4 total, 3 up, 4 in 2024-03-29T04:48:23.368 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:22 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:48:23.368 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:22 smithi181 ceph-mon[41183]: from='osd.3 [v2:172.21.15.181:6826/1785806254,v1:172.21.15.181:6827/1785806254]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:48:23.368 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:22 smithi181 ceph-mon[41183]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:48:23.369 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:22 smithi181 ceph-mon[29765]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T04:48:23.369 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:22 smithi181 ceph-mon[29765]: osdmap e24: 4 total, 3 up, 4 in 2024-03-29T04:48:23.369 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:22 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:48:23.369 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:22 smithi181 ceph-mon[29765]: from='osd.3 [v2:172.21.15.181:6826/1785806254,v1:172.21.15.181:6827/1785806254]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:48:23.369 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:22 smithi181 ceph-mon[29765]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:48:23.369 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 04:48:22 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3[64411]: 2024-03-29T04:48:22.984+0000 7f78c325e700 -1 osd.3 0 waiting for initial osdmap 2024-03-29T04:48:23.369 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 04:48:22 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3[64411]: 2024-03-29T04:48:22.988+0000 7f78bc856700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:48:24.127 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:23 smithi187 ceph-mon[33595]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-03-29T04:48:24.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:23 smithi187 ceph-mon[33595]: osdmap e25: 4 total, 3 up, 4 in 2024-03-29T04:48:24.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:23 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:48:24.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:23 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:48:24.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:23 smithi187 ceph-mon[33595]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:24.335 INFO:teuthology.orchestra.run.smithi187.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-29T04:48:24.335 INFO:teuthology.orchestra.run.smithi187.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-29T04:48:24.336 INFO:teuthology.orchestra.run.smithi187.stderr: stderr: 10+0 records in 2024-03-29T04:48:24.336 INFO:teuthology.orchestra.run.smithi187.stderr:10+0 records out 2024-03-29T04:48:24.336 INFO:teuthology.orchestra.run.smithi187.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0112785 s, 930 MB/s 2024-03-29T04:48:24.336 INFO:teuthology.orchestra.run.smithi187.stderr: stderr: 2024-03-29T04:48:24.336 INFO:teuthology.orchestra.run.smithi187.stderr:--> Zapping successful for: 2024-03-29T04:48:24.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:23 smithi181 ceph-mon[29765]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-03-29T04:48:24.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:23 smithi181 ceph-mon[29765]: osdmap e25: 4 total, 3 up, 4 in 2024-03-29T04:48:24.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:23 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:48:24.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:23 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:48:24.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:23 smithi181 ceph-mon[29765]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:24.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:23 smithi181 ceph-mon[41183]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]': finished 2024-03-29T04:48:24.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:23 smithi181 ceph-mon[41183]: osdmap e25: 4 total, 3 up, 4 in 2024-03-29T04:48:24.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:23 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:48:24.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:23 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:48:24.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:23 smithi181 ceph-mon[41183]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 875 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:48:24.940 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi187:vg_nvme/lv_4 2024-03-29T04:48:25.092 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:24 smithi181 ceph-mon[29765]: purged_snaps scrub starts 2024-03-29T04:48:25.092 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:24 smithi181 ceph-mon[29765]: purged_snaps scrub ok 2024-03-29T04:48:25.092 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:24 smithi181 ceph-mon[29765]: osd.3 [v2:172.21.15.181:6826/1785806254,v1:172.21.15.181:6827/1785806254] boot 2024-03-29T04:48:25.092 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:24 smithi181 ceph-mon[29765]: osdmap e26: 4 total, 4 up, 4 in 2024-03-29T04:48:25.092 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:24 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:48:25.092 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:24 smithi181 ceph-mon[41183]: purged_snaps scrub starts 2024-03-29T04:48:25.093 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:24 smithi181 ceph-mon[41183]: purged_snaps scrub ok 2024-03-29T04:48:25.093 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:24 smithi181 ceph-mon[41183]: osd.3 [v2:172.21.15.181:6826/1785806254,v1:172.21.15.181:6827/1785806254] boot 2024-03-29T04:48:25.093 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:24 smithi181 ceph-mon[41183]: osdmap e26: 4 total, 4 up, 4 in 2024-03-29T04:48:25.093 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:24 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:48:25.229 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:48:25.351 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:24 smithi187 ceph-mon[33595]: purged_snaps scrub starts 2024-03-29T04:48:25.351 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:24 smithi187 ceph-mon[33595]: purged_snaps scrub ok 2024-03-29T04:48:25.351 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:24 smithi187 ceph-mon[33595]: osd.3 [v2:172.21.15.181:6826/1785806254,v1:172.21.15.181:6827/1785806254] boot 2024-03-29T04:48:25.351 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:24 smithi187 ceph-mon[33595]: osdmap e26: 4 total, 4 up, 4 in 2024-03-29T04:48:25.351 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:24 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:48:26.256 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: osdmap e27: 4 total, 4 up, 4 in 2024-03-29T04:48:26.256 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: Detected new or changed devices on smithi181 2024-03-29T04:48:26.256 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:26.256 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:26.256 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:26.256 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:26.256 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:26.256 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:26.256 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: Adjusting osd_memory_target on smithi181 to 4016M 2024-03-29T04:48:26.256 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:26.256 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:26.256 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:26.256 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:26.257 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:26.257 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:26.257 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:26.257 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:26.257 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:25 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:26.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: osdmap e27: 4 total, 4 up, 4 in 2024-03-29T04:48:26.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: Detected new or changed devices on smithi181 2024-03-29T04:48:26.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:26.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:26.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:26.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:26.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:26.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:26.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: Adjusting osd_memory_target on smithi181 to 4016M 2024-03-29T04:48:26.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:26.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:26.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:26.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:26.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:26.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:26.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:26.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:26.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:26.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: osdmap e27: 4 total, 4 up, 4 in 2024-03-29T04:48:26.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: Detected new or changed devices on smithi181 2024-03-29T04:48:26.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:26.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:26.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:26.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:26.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:26.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:26.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: Adjusting osd_memory_target on smithi181 to 4016M 2024-03-29T04:48:26.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:26.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:26.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:26.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:26.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:26.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:26.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:26.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:26.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:25 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:27.498 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:27 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:48:27.498 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:27 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:48:27.498 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:27 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:27.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:27 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:48:27.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:27 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:48:27.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:27 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:27.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:27 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:48:27.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:27 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:48:27.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:27 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:28.377 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:28 smithi187 ceph-mon[33595]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:28.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:28 smithi187 ceph-mon[33595]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:28.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:28 smithi181 ceph-mon[41183]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:28.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:28 smithi181 ceph-mon[41183]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:28.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:28 smithi181 ceph-mon[29765]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:28.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:28 smithi181 ceph-mon[29765]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:30.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:30 smithi187 ceph-mon[33595]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:30.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:30 smithi187 ceph-mon[33595]: from='client.? 172.21.15.187:0/964235857' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a71f228-2cc8-4b83-b86d-62a2854a8533"}]: dispatch 2024-03-29T04:48:30.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:30 smithi187 ceph-mon[33595]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a71f228-2cc8-4b83-b86d-62a2854a8533"}]: dispatch 2024-03-29T04:48:30.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:30 smithi187 ceph-mon[33595]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0a71f228-2cc8-4b83-b86d-62a2854a8533"}]': finished 2024-03-29T04:48:30.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:30 smithi187 ceph-mon[33595]: osdmap e28: 5 total, 4 up, 5 in 2024-03-29T04:48:30.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:30 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:48:30.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:30 smithi187 ceph-mon[33595]: from='client.? 172.21.15.187:0/4168308656' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:48:30.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:30 smithi181 ceph-mon[41183]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:30.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:30 smithi181 ceph-mon[41183]: from='client.? 172.21.15.187:0/964235857' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a71f228-2cc8-4b83-b86d-62a2854a8533"}]: dispatch 2024-03-29T04:48:30.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:30 smithi181 ceph-mon[41183]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a71f228-2cc8-4b83-b86d-62a2854a8533"}]: dispatch 2024-03-29T04:48:30.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:30 smithi181 ceph-mon[41183]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0a71f228-2cc8-4b83-b86d-62a2854a8533"}]': finished 2024-03-29T04:48:30.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:30 smithi181 ceph-mon[41183]: osdmap e28: 5 total, 4 up, 5 in 2024-03-29T04:48:30.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:30 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:48:30.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:30 smithi181 ceph-mon[41183]: from='client.? 172.21.15.187:0/4168308656' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:48:30.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:30 smithi181 ceph-mon[29765]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:30.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:30 smithi181 ceph-mon[29765]: from='client.? 172.21.15.187:0/964235857' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a71f228-2cc8-4b83-b86d-62a2854a8533"}]: dispatch 2024-03-29T04:48:30.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:30 smithi181 ceph-mon[29765]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a71f228-2cc8-4b83-b86d-62a2854a8533"}]: dispatch 2024-03-29T04:48:30.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:30 smithi181 ceph-mon[29765]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0a71f228-2cc8-4b83-b86d-62a2854a8533"}]': finished 2024-03-29T04:48:30.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:30 smithi181 ceph-mon[29765]: osdmap e28: 5 total, 4 up, 5 in 2024-03-29T04:48:30.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:30 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:48:30.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:30 smithi181 ceph-mon[29765]: from='client.? 172.21.15.187:0/4168308656' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:48:32.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:32 smithi187 ceph-mon[33595]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:32.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:32 smithi181 ceph-mon[41183]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:32.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:32 smithi181 ceph-mon[29765]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:34.538 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:34 smithi187 ceph-mon[33595]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:34.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:34 smithi181 ceph-mon[41183]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:34.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:34 smithi181 ceph-mon[29765]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:36.556 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:36 smithi187 ceph-mon[33595]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:36.556 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:36 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:48:36.556 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:36 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:36.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:36 smithi181 ceph-mon[41183]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:36.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:36 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:48:36.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:36 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:36.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:36 smithi181 ceph-mon[29765]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:36.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:36 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:48:36.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:36 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:37.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:37 smithi187 ceph-mon[33595]: Deploying daemon osd.4 on smithi187 2024-03-29T04:48:37.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:37 smithi181 ceph-mon[41183]: Deploying daemon osd.4 on smithi187 2024-03-29T04:48:37.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:37 smithi181 ceph-mon[29765]: Deploying daemon osd.4 on smithi187 2024-03-29T04:48:38.516 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:38 smithi187 ceph-mon[33595]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:38.599 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:38 smithi181 ceph-mon[41183]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:38.599 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:38 smithi181 ceph-mon[29765]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:40.515 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:40 smithi187 ceph-mon[33595]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:40.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:40 smithi181 ceph-mon[29765]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:40.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:40 smithi181 ceph-mon[41183]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:42.105 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:42 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:42.105 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:42 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:42.105 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:42 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:42.105 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:42 smithi187 ceph-mon[33595]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:42.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:42 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:42.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:42 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:42.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:42 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:42.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:42 smithi181 ceph-mon[29765]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:42.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:42 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:42.448 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:42 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:42.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:42 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:42.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:42 smithi181 ceph-mon[41183]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:44.518 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:44 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:44.518 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:44 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:44.518 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:44 smithi187 ceph-mon[33595]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:44.518 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:44 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:44.518 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:44 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:44.518 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:44 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:44.518 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:44 smithi187 ceph-mon[33595]: from='osd.4 [v2:172.21.15.187:6800/3700510703,v1:172.21.15.187:6801/3700510703]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:48:44.518 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:44 smithi187 ceph-mon[33595]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:48:44.581 INFO:teuthology.orchestra.run.smithi187.stdout:Created osd(s) 4 on host 'smithi187' 2024-03-29T04:48:44.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:44 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:44.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:44 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:44.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:44 smithi181 ceph-mon[41183]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:44.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:44 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:44.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:44 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:44.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:44 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:44.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:44 smithi181 ceph-mon[41183]: from='osd.4 [v2:172.21.15.187:6800/3700510703,v1:172.21.15.187:6801/3700510703]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:48:44.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:44 smithi181 ceph-mon[41183]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:48:44.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:44 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:44.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:44 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:44.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:44 smithi181 ceph-mon[29765]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:44.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:44 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:44.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:44 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:44.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:44 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:44.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:44 smithi181 ceph-mon[29765]: from='osd.4 [v2:172.21.15.187:6800/3700510703,v1:172.21.15.187:6801/3700510703]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:48:44.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:44 smithi181 ceph-mon[29765]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:48:45.287 DEBUG:teuthology.orchestra.run.smithi187:osd.4> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.4.service 2024-03-29T04:48:45.289 INFO:tasks.cephadm:Deploying osd.5 on smithi187 with /dev/vg_nvme/lv_3... 2024-03-29T04:48:45.289 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-29T04:48:45.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:45 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:45.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:45 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:45.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:45 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:45.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:45 smithi187 ceph-mon[33595]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-29T04:48:45.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:45 smithi187 ceph-mon[33595]: osdmap e29: 5 total, 4 up, 5 in 2024-03-29T04:48:45.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:45 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:48:45.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:45 smithi187 ceph-mon[33595]: from='osd.4 [v2:172.21.15.187:6800/3700510703,v1:172.21.15.187:6801/3700510703]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:48:45.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:45 smithi187 ceph-mon[33595]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:48:45.879 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 04:48:45 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[39028]: 2024-03-29T04:48:45.627+0000 7f0f10ee5700 -1 osd.4 0 waiting for initial osdmap 2024-03-29T04:48:45.879 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 04:48:45 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[39028]: 2024-03-29T04:48:45.633+0000 7f0f09cdc700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:48:45.940 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:48:45.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:45 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:45.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:45 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:45.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:45 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:45.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:45 smithi181 ceph-mon[29765]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-29T04:48:45.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:45 smithi181 ceph-mon[29765]: osdmap e29: 5 total, 4 up, 5 in 2024-03-29T04:48:45.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:45 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:48:45.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:45 smithi181 ceph-mon[29765]: from='osd.4 [v2:172.21.15.187:6800/3700510703,v1:172.21.15.187:6801/3700510703]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:48:45.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:45 smithi181 ceph-mon[29765]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:48:45.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:45 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:45.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:45 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:45.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:45 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:45.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:45 smithi181 ceph-mon[41183]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-29T04:48:45.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:45 smithi181 ceph-mon[41183]: osdmap e29: 5 total, 4 up, 5 in 2024-03-29T04:48:45.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:45 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:48:45.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:45 smithi181 ceph-mon[41183]: from='osd.4 [v2:172.21.15.187:6800/3700510703,v1:172.21.15.187:6801/3700510703]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:48:45.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:45 smithi181 ceph-mon[41183]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:48:46.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:46 smithi181 ceph-mon[29765]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:46.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:46 smithi181 ceph-mon[29765]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-29T04:48:46.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:46 smithi181 ceph-mon[29765]: osdmap e30: 5 total, 4 up, 5 in 2024-03-29T04:48:46.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:46 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:48:46.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:46 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:48:46.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:46 smithi181 ceph-mon[41183]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:46.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:46 smithi181 ceph-mon[41183]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-29T04:48:46.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:46 smithi181 ceph-mon[41183]: osdmap e30: 5 total, 4 up, 5 in 2024-03-29T04:48:46.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:46 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:48:46.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:46 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:48:46.974 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:46 smithi187 ceph-mon[33595]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:48:46.975 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:46 smithi187 ceph-mon[33595]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-29T04:48:46.975 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:46 smithi187 ceph-mon[33595]: osdmap e30: 5 total, 4 up, 5 in 2024-03-29T04:48:46.975 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:46 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:48:46.975 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:46 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:48:47.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:47 smithi187 ceph-mon[33595]: purged_snaps scrub starts 2024-03-29T04:48:47.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:47 smithi187 ceph-mon[33595]: purged_snaps scrub ok 2024-03-29T04:48:47.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:47 smithi187 ceph-mon[33595]: osd.4 [v2:172.21.15.187:6800/3700510703,v1:172.21.15.187:6801/3700510703] boot 2024-03-29T04:48:47.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:47 smithi187 ceph-mon[33595]: osdmap e31: 5 total, 5 up, 5 in 2024-03-29T04:48:47.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:47 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:48:47.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:47 smithi187 ceph-mon[33595]: osdmap e32: 5 total, 5 up, 5 in 2024-03-29T04:48:47.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:47 smithi181 ceph-mon[29765]: purged_snaps scrub starts 2024-03-29T04:48:47.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:47 smithi181 ceph-mon[29765]: purged_snaps scrub ok 2024-03-29T04:48:47.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:47 smithi181 ceph-mon[29765]: osd.4 [v2:172.21.15.187:6800/3700510703,v1:172.21.15.187:6801/3700510703] boot 2024-03-29T04:48:47.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:47 smithi181 ceph-mon[29765]: osdmap e31: 5 total, 5 up, 5 in 2024-03-29T04:48:47.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:47 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:48:47.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:47 smithi181 ceph-mon[29765]: osdmap e32: 5 total, 5 up, 5 in 2024-03-29T04:48:47.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:47 smithi181 ceph-mon[41183]: purged_snaps scrub starts 2024-03-29T04:48:47.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:47 smithi181 ceph-mon[41183]: purged_snaps scrub ok 2024-03-29T04:48:47.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:47 smithi181 ceph-mon[41183]: osd.4 [v2:172.21.15.187:6800/3700510703,v1:172.21.15.187:6801/3700510703] boot 2024-03-29T04:48:47.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:47 smithi181 ceph-mon[41183]: osdmap e31: 5 total, 5 up, 5 in 2024-03-29T04:48:47.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:47 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:48:47.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:47 smithi181 ceph-mon[41183]: osdmap e32: 5 total, 5 up, 5 in 2024-03-29T04:48:48.844 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:48 smithi187 ceph-mon[33595]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:48:48.844 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:48 smithi187 ceph-mon[33595]: osdmap e33: 5 total, 5 up, 5 in 2024-03-29T04:48:48.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:48 smithi181 ceph-mon[29765]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:48:49.183 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:48 smithi181 ceph-mon[29765]: osdmap e33: 5 total, 5 up, 5 in 2024-03-29T04:48:49.183 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:48 smithi181 ceph-mon[41183]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:48:49.183 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:48 smithi181 ceph-mon[41183]: osdmap e33: 5 total, 5 up, 5 in 2024-03-29T04:48:49.433 INFO:teuthology.orchestra.run.smithi187.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-29T04:48:49.434 INFO:teuthology.orchestra.run.smithi187.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-29T04:48:49.434 INFO:teuthology.orchestra.run.smithi187.stderr: stderr: 10+0 records in 2024-03-29T04:48:49.434 INFO:teuthology.orchestra.run.smithi187.stderr:10+0 records out 2024-03-29T04:48:49.434 INFO:teuthology.orchestra.run.smithi187.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112815 s, 929 MB/s 2024-03-29T04:48:49.434 INFO:teuthology.orchestra.run.smithi187.stderr:--> Zapping successful for: 2024-03-29T04:48:50.124 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:49 smithi187 ceph-mon[33595]: Detected new or changed devices on smithi187 2024-03-29T04:48:50.124 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:49 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:50.124 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:49 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:50.124 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:49 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:50.124 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:49 smithi187 ceph-mon[33595]: Adjusting osd_memory_target on smithi187 to 17088M 2024-03-29T04:48:50.124 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:49 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:50.124 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:49 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:50.124 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:49 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:50.125 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:49 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:50.125 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:49 smithi187 ceph-mon[33595]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:48:50.125 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:49 smithi187 ceph-mon[33595]: osdmap e34: 5 total, 5 up, 5 in 2024-03-29T04:48:50.150 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi187:vg_nvme/lv_3 2024-03-29T04:48:50.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[29765]: Detected new or changed devices on smithi187 2024-03-29T04:48:50.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:50.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:50.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:50.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[29765]: Adjusting osd_memory_target on smithi187 to 17088M 2024-03-29T04:48:50.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:50.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:50.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:50.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:50.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[29765]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:48:50.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[29765]: osdmap e34: 5 total, 5 up, 5 in 2024-03-29T04:48:50.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[41183]: Detected new or changed devices on smithi187 2024-03-29T04:48:50.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:50.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:50.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:48:50.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[41183]: Adjusting osd_memory_target on smithi187 to 17088M 2024-03-29T04:48:50.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:50.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:50.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:50.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:48:50.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[41183]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:48:50.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:49 smithi181 ceph-mon[41183]: osdmap e34: 5 total, 5 up, 5 in 2024-03-29T04:48:50.398 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:48:51.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:51 smithi187 ceph-mon[33595]: osdmap e35: 5 total, 5 up, 5 in 2024-03-29T04:48:51.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:51 smithi181 ceph-mon[29765]: osdmap e35: 5 total, 5 up, 5 in 2024-03-29T04:48:51.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:51 smithi181 ceph-mon[41183]: osdmap e35: 5 total, 5 up, 5 in 2024-03-29T04:48:52.675 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:52 smithi187 ceph-mon[33595]: pgmap v116: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-03-29T04:48:52.675 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:52 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:48:52.675 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:52 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:48:52.676 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:52 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:52.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:52 smithi181 ceph-mon[29765]: pgmap v116: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-03-29T04:48:52.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:52 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:48:52.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:52 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:48:52.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:52 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:52.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:52 smithi181 ceph-mon[41183]: pgmap v116: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-03-29T04:48:52.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:52 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:48:52.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:52 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:48:52.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:52 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:53.564 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:53 smithi187 ceph-mon[33595]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:53.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:53 smithi181 ceph-mon[29765]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:53.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:53 smithi181 ceph-mon[41183]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:54.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:54 smithi181 ceph-mon[41183]: pgmap v117: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-03-29T04:48:54.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:54 smithi181 ceph-mon[29765]: pgmap v117: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-03-29T04:48:54.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:54 smithi187 ceph-mon[33595]: pgmap v117: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-03-29T04:48:55.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:55 smithi181 ceph-mon[29765]: from='client.? 172.21.15.187:0/4179962247' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "847657e4-7b9a-4eee-9df5-8e32d0d6543e"}]: dispatch 2024-03-29T04:48:55.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:55 smithi181 ceph-mon[29765]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "847657e4-7b9a-4eee-9df5-8e32d0d6543e"}]: dispatch 2024-03-29T04:48:55.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:55 smithi181 ceph-mon[29765]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "847657e4-7b9a-4eee-9df5-8e32d0d6543e"}]': finished 2024-03-29T04:48:55.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:55 smithi181 ceph-mon[29765]: osdmap e36: 6 total, 5 up, 6 in 2024-03-29T04:48:55.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:55 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:48:55.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:55 smithi181 ceph-mon[29765]: from='client.? 172.21.15.187:0/2845885944' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:48:55.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:55 smithi181 ceph-mon[41183]: from='client.? 172.21.15.187:0/4179962247' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "847657e4-7b9a-4eee-9df5-8e32d0d6543e"}]: dispatch 2024-03-29T04:48:55.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:55 smithi181 ceph-mon[41183]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "847657e4-7b9a-4eee-9df5-8e32d0d6543e"}]: dispatch 2024-03-29T04:48:55.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:55 smithi181 ceph-mon[41183]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "847657e4-7b9a-4eee-9df5-8e32d0d6543e"}]': finished 2024-03-29T04:48:55.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:55 smithi181 ceph-mon[41183]: osdmap e36: 6 total, 5 up, 6 in 2024-03-29T04:48:55.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:55 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:48:55.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:55 smithi181 ceph-mon[41183]: from='client.? 172.21.15.187:0/2845885944' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:48:55.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:55 smithi187 ceph-mon[33595]: from='client.? 172.21.15.187:0/4179962247' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "847657e4-7b9a-4eee-9df5-8e32d0d6543e"}]: dispatch 2024-03-29T04:48:55.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:55 smithi187 ceph-mon[33595]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "847657e4-7b9a-4eee-9df5-8e32d0d6543e"}]: dispatch 2024-03-29T04:48:55.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:55 smithi187 ceph-mon[33595]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "847657e4-7b9a-4eee-9df5-8e32d0d6543e"}]': finished 2024-03-29T04:48:55.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:55 smithi187 ceph-mon[33595]: osdmap e36: 6 total, 5 up, 6 in 2024-03-29T04:48:55.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:55 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:48:55.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:55 smithi187 ceph-mon[33595]: from='client.? 172.21.15.187:0/2845885944' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:48:56.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:56 smithi181 ceph-mon[29765]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:48:56.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:56 smithi181 ceph-mon[41183]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:48:57.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:56 smithi187 ceph-mon[33595]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:48:59.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:48:58 smithi187 ceph-mon[33595]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:48:59.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:48:58 smithi181 ceph-mon[29765]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:48:59.198 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:48:58 smithi181 ceph-mon[41183]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:49:00.995 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:00 smithi187 ceph-mon[33595]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:01.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:00 smithi181 ceph-mon[29765]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:01.198 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:00 smithi181 ceph-mon[41183]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:02.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:01 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:49:02.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:01 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:02.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:01 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:49:02.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:01 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:02.198 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:01 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:49:02.198 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:01 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:03.068 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:02 smithi187 ceph-mon[33595]: Deploying daemon osd.5 on smithi187 2024-03-29T04:49:03.068 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:02 smithi187 ceph-mon[33595]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:03.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:02 smithi181 ceph-mon[29765]: Deploying daemon osd.5 on smithi187 2024-03-29T04:49:03.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:02 smithi181 ceph-mon[29765]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:03.198 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:02 smithi181 ceph-mon[41183]: Deploying daemon osd.5 on smithi187 2024-03-29T04:49:03.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:02 smithi181 ceph-mon[41183]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:04.760 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:04 smithi187 ceph-mon[33595]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:05.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:04 smithi181 ceph-mon[29765]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:05.198 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:04 smithi181 ceph-mon[41183]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:07.055 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:06 smithi187 ceph-mon[33595]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:07.055 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:06 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:07.055 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:06 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:07.055 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:06 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:07.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:06 smithi181 ceph-mon[29765]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:07.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:06 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:07.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:06 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:07.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:06 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:07.198 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:06 smithi181 ceph-mon[41183]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:07.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:06 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:07.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:06 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:07.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:06 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:09.000 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:08 smithi181 ceph-mon[29765]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:09.000 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:08 smithi181 ceph-mon[41183]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:09.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:08 smithi187 ceph-mon[33595]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:09.454 INFO:teuthology.orchestra.run.smithi187.stdout:Created osd(s) 5 on host 'smithi187' 2024-03-29T04:49:10.129 DEBUG:teuthology.orchestra.run.smithi187:osd.5> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.5.service 2024-03-29T04:49:10.131 INFO:tasks.cephadm:Deploying osd.6 on smithi187 with /dev/vg_nvme/lv_2... 2024-03-29T04:49:10.131 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-29T04:49:10.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:10 smithi187 ceph-mon[33595]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:10.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:10 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:10.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:10 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:10.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:10 smithi187 ceph-mon[33595]: from='osd.5 [v2:172.21.15.187:6808/989620923,v1:172.21.15.187:6809/989620923]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:49:10.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:10 smithi187 ceph-mon[33595]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:49:10.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:10 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:10.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:10 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:10.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:10 smithi181 ceph-mon[29765]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:10.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:10 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:10.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:10 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:10.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:10 smithi181 ceph-mon[29765]: from='osd.5 [v2:172.21.15.187:6808/989620923,v1:172.21.15.187:6809/989620923]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:49:10.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:10 smithi181 ceph-mon[29765]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:49:10.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:10 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:10.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:10 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:10.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:10 smithi181 ceph-mon[41183]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:10.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:10 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:10.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:10 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:10.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:10 smithi181 ceph-mon[41183]: from='osd.5 [v2:172.21.15.187:6808/989620923,v1:172.21.15.187:6809/989620923]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:49:10.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:10 smithi181 ceph-mon[41183]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:49:10.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:10 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:10.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:10 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:10.965 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:49:11.798 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:11 smithi187 ceph-mon[33595]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-29T04:49:11.798 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:11 smithi187 ceph-mon[33595]: osdmap e37: 6 total, 5 up, 6 in 2024-03-29T04:49:11.798 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:11 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:49:11.801 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:11 smithi187 ceph-mon[33595]: from='osd.5 [v2:172.21.15.187:6808/989620923,v1:172.21.15.187:6809/989620923]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:49:11.801 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:11 smithi187 ceph-mon[33595]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:49:11.802 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 04:49:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T04:49:11.459+0000 7f67d3376700 -1 osd.5 0 waiting for initial osdmap 2024-03-29T04:49:11.802 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 04:49:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T04:49:11.464+0000 7f67ca96a700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:49:11.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:11 smithi181 ceph-mon[29765]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-29T04:49:11.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:11 smithi181 ceph-mon[29765]: osdmap e37: 6 total, 5 up, 6 in 2024-03-29T04:49:11.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:11 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:49:11.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:11 smithi181 ceph-mon[29765]: from='osd.5 [v2:172.21.15.187:6808/989620923,v1:172.21.15.187:6809/989620923]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:49:11.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:11 smithi181 ceph-mon[29765]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:49:11.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:11 smithi181 ceph-mon[41183]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-29T04:49:11.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:11 smithi181 ceph-mon[41183]: osdmap e37: 6 total, 5 up, 6 in 2024-03-29T04:49:11.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:11 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:49:11.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:11 smithi181 ceph-mon[41183]: from='osd.5 [v2:172.21.15.187:6808/989620923,v1:172.21.15.187:6809/989620923]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:49:11.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:11 smithi181 ceph-mon[41183]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:49:12.684 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:12 smithi187 ceph-mon[33595]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:12.685 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:12 smithi187 ceph-mon[33595]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-29T04:49:12.685 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:12 smithi187 ceph-mon[33595]: osdmap e38: 6 total, 5 up, 6 in 2024-03-29T04:49:12.685 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:12 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:49:12.685 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:12 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:49:12.685 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:12 smithi187 ceph-mon[33595]: osd.5 [v2:172.21.15.187:6808/989620923,v1:172.21.15.187:6809/989620923] boot 2024-03-29T04:49:12.685 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:12 smithi187 ceph-mon[33595]: osdmap e39: 6 total, 6 up, 6 in 2024-03-29T04:49:12.685 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:12 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:49:12.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:12 smithi181 ceph-mon[29765]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:12.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:12 smithi181 ceph-mon[29765]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-29T04:49:12.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:12 smithi181 ceph-mon[29765]: osdmap e38: 6 total, 5 up, 6 in 2024-03-29T04:49:12.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:12 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:49:12.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:12 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:49:12.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:12 smithi181 ceph-mon[29765]: osd.5 [v2:172.21.15.187:6808/989620923,v1:172.21.15.187:6809/989620923] boot 2024-03-29T04:49:12.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:12 smithi181 ceph-mon[29765]: osdmap e39: 6 total, 6 up, 6 in 2024-03-29T04:49:12.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:12 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:49:12.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:12 smithi181 ceph-mon[41183]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:49:12.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:12 smithi181 ceph-mon[41183]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-29T04:49:12.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:12 smithi181 ceph-mon[41183]: osdmap e38: 6 total, 5 up, 6 in 2024-03-29T04:49:12.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:12 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:49:12.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:12 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:49:12.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:12 smithi181 ceph-mon[41183]: osd.5 [v2:172.21.15.187:6808/989620923,v1:172.21.15.187:6809/989620923] boot 2024-03-29T04:49:12.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:12 smithi181 ceph-mon[41183]: osdmap e39: 6 total, 6 up, 6 in 2024-03-29T04:49:12.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:12 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:49:13.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:13 smithi187 ceph-mon[33595]: purged_snaps scrub starts 2024-03-29T04:49:13.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:13 smithi187 ceph-mon[33595]: purged_snaps scrub ok 2024-03-29T04:49:13.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:13 smithi187 ceph-mon[33595]: osdmap e40: 6 total, 6 up, 6 in 2024-03-29T04:49:13.685 INFO:teuthology.orchestra.run.smithi187.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-29T04:49:13.686 INFO:teuthology.orchestra.run.smithi187.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-29T04:49:13.686 INFO:teuthology.orchestra.run.smithi187.stderr: stderr: 10+0 records in 2024-03-29T04:49:13.686 INFO:teuthology.orchestra.run.smithi187.stderr:10+0 records out 2024-03-29T04:49:13.686 INFO:teuthology.orchestra.run.smithi187.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0155878 s, 673 MB/s 2024-03-29T04:49:13.686 INFO:teuthology.orchestra.run.smithi187.stderr:--> Zapping successful for: 2024-03-29T04:49:13.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:13 smithi181 ceph-mon[29765]: purged_snaps scrub starts 2024-03-29T04:49:13.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:13 smithi181 ceph-mon[29765]: purged_snaps scrub ok 2024-03-29T04:49:13.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:13 smithi181 ceph-mon[29765]: osdmap e40: 6 total, 6 up, 6 in 2024-03-29T04:49:13.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:13 smithi181 ceph-mon[41183]: purged_snaps scrub starts 2024-03-29T04:49:13.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:13 smithi181 ceph-mon[41183]: purged_snaps scrub ok 2024-03-29T04:49:13.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:13 smithi181 ceph-mon[41183]: osdmap e40: 6 total, 6 up, 6 in 2024-03-29T04:49:14.402 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi187:vg_nvme/lv_2 2024-03-29T04:49:14.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:14 smithi187 ceph-mon[33595]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:14.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:14 smithi187 ceph-mon[33595]: osdmap e41: 6 total, 6 up, 6 in 2024-03-29T04:49:14.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:14 smithi181 ceph-mon[29765]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:14.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:14 smithi181 ceph-mon[29765]: osdmap e41: 6 total, 6 up, 6 in 2024-03-29T04:49:14.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:14 smithi181 ceph-mon[41183]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:14.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:14 smithi181 ceph-mon[41183]: osdmap e41: 6 total, 6 up, 6 in 2024-03-29T04:49:14.987 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:49:16.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:15 smithi187 ceph-mon[33595]: Detected new or changed devices on smithi187 2024-03-29T04:49:16.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:15 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:16.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:15 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:16.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:15 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:49:16.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:15 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:49:16.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:15 smithi187 ceph-mon[33595]: Adjusting osd_memory_target on smithi187 to 8544M 2024-03-29T04:49:16.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:15 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:16.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:15 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:16.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:15 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:16.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:15 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:16.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:15 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:16.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:15 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:16.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:15 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:16.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:15 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:16.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:15 smithi187 ceph-mon[33595]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:16.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:15 smithi187 ceph-mon[33595]: osdmap e42: 6 total, 6 up, 6 in 2024-03-29T04:49:16.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[29765]: Detected new or changed devices on smithi187 2024-03-29T04:49:16.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:16.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:16.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:49:16.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:49:16.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[29765]: Adjusting osd_memory_target on smithi187 to 8544M 2024-03-29T04:49:16.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:16.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:16.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:16.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:16.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:16.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:16.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:16.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:16.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[29765]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:16.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[29765]: osdmap e42: 6 total, 6 up, 6 in 2024-03-29T04:49:16.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[41183]: Detected new or changed devices on smithi187 2024-03-29T04:49:16.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:16.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:16.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:49:16.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:49:16.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[41183]: Adjusting osd_memory_target on smithi187 to 8544M 2024-03-29T04:49:16.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:16.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:16.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:16.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:16.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:16.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:16.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:16.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:16.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[41183]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:16.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:15 smithi181 ceph-mon[41183]: osdmap e42: 6 total, 6 up, 6 in 2024-03-29T04:49:17.481 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:17 smithi187 ceph-mon[33595]: osdmap e43: 6 total, 6 up, 6 in 2024-03-29T04:49:17.482 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:17 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:49:17.482 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:17 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:49:17.482 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:17 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:17.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:17 smithi181 ceph-mon[41183]: osdmap e43: 6 total, 6 up, 6 in 2024-03-29T04:49:17.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:17 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:49:17.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:17 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:49:17.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:17 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:17.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:17 smithi181 ceph-mon[29765]: osdmap e43: 6 total, 6 up, 6 in 2024-03-29T04:49:17.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:17 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:49:17.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:17 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:49:17.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:17 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:18.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:18 smithi187 ceph-mon[33595]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:49:18.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:18 smithi187 ceph-mon[33595]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:18.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:18 smithi181 ceph-mon[41183]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:49:18.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:18 smithi181 ceph-mon[41183]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:18.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:18 smithi181 ceph-mon[29765]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:49:18.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:18 smithi181 ceph-mon[29765]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:19.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:19 smithi187 ceph-mon[33595]: from='client.? 172.21.15.187:0/337350519' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cead714e-ffc1-44a4-bef5-cfa772070c9f"}]: dispatch 2024-03-29T04:49:19.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:19 smithi187 ceph-mon[33595]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cead714e-ffc1-44a4-bef5-cfa772070c9f"}]: dispatch 2024-03-29T04:49:19.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:19 smithi187 ceph-mon[33595]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cead714e-ffc1-44a4-bef5-cfa772070c9f"}]': finished 2024-03-29T04:49:19.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:19 smithi187 ceph-mon[33595]: osdmap e44: 7 total, 6 up, 7 in 2024-03-29T04:49:19.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:19 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:49:19.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:19 smithi181 ceph-mon[41183]: from='client.? 172.21.15.187:0/337350519' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cead714e-ffc1-44a4-bef5-cfa772070c9f"}]: dispatch 2024-03-29T04:49:19.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:19 smithi181 ceph-mon[41183]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cead714e-ffc1-44a4-bef5-cfa772070c9f"}]: dispatch 2024-03-29T04:49:19.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:19 smithi181 ceph-mon[41183]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cead714e-ffc1-44a4-bef5-cfa772070c9f"}]': finished 2024-03-29T04:49:19.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:19 smithi181 ceph-mon[41183]: osdmap e44: 7 total, 6 up, 7 in 2024-03-29T04:49:19.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:19 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:49:19.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:19 smithi181 ceph-mon[29765]: from='client.? 172.21.15.187:0/337350519' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cead714e-ffc1-44a4-bef5-cfa772070c9f"}]: dispatch 2024-03-29T04:49:19.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:19 smithi181 ceph-mon[29765]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cead714e-ffc1-44a4-bef5-cfa772070c9f"}]: dispatch 2024-03-29T04:49:19.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:19 smithi181 ceph-mon[29765]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cead714e-ffc1-44a4-bef5-cfa772070c9f"}]': finished 2024-03-29T04:49:19.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:19 smithi181 ceph-mon[29765]: osdmap e44: 7 total, 6 up, 7 in 2024-03-29T04:49:19.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:19 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:49:20.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:20 smithi187 ceph-mon[33595]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:20.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:20 smithi187 ceph-mon[33595]: from='client.? 172.21.15.187:0/4155503702' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:49:20.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:20 smithi181 ceph-mon[41183]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:20.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:20 smithi181 ceph-mon[41183]: from='client.? 172.21.15.187:0/4155503702' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:49:20.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:20 smithi181 ceph-mon[29765]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:20.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:20 smithi181 ceph-mon[29765]: from='client.? 172.21.15.187:0/4155503702' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:49:22.738 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:22 smithi187 ceph-mon[33595]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:22.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:22 smithi181 ceph-mon[41183]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:22.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:22 smithi181 ceph-mon[29765]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:24.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:24 smithi187 ceph-mon[33595]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:24.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:24 smithi181 ceph-mon[41183]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:24.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:24 smithi181 ceph-mon[29765]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:25.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:25 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:49:25.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:25 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:25.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:25 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:49:25.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:25 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:25.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:25 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:49:25.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:25 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:26.487 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:26 smithi187 ceph-mon[33595]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:26.487 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:26 smithi187 ceph-mon[33595]: Deploying daemon osd.6 on smithi187 2024-03-29T04:49:26.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:26 smithi181 ceph-mon[29765]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:26.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:26 smithi181 ceph-mon[29765]: Deploying daemon osd.6 on smithi187 2024-03-29T04:49:26.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:26 smithi181 ceph-mon[41183]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:26.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:26 smithi181 ceph-mon[41183]: Deploying daemon osd.6 on smithi187 2024-03-29T04:49:28.628 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:28 smithi187 ceph-mon[33595]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:28.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:28 smithi181 ceph-mon[29765]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:28.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:28 smithi181 ceph-mon[41183]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:30.808 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:30 smithi187 ceph-mon[33595]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:30.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:30 smithi181 ceph-mon[29765]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:30.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:30 smithi181 ceph-mon[41183]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:32.088 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:31 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:32.088 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:31 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:32.088 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:31 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:32.198 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:31 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:32.198 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:31 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:32.198 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:31 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:32.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:31 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:32.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:31 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:32.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:31 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:33.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:32 smithi187 ceph-mon[33595]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:33.198 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:32 smithi181 ceph-mon[41183]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:33.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:32 smithi181 ceph-mon[29765]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:34.429 INFO:teuthology.orchestra.run.smithi187.stdout:Created osd(s) 6 on host 'smithi187' 2024-03-29T04:49:34.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:34 smithi187 ceph-mon[33595]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:34.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:34 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:34.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:34 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:34.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:34 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:34.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:34 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:34.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:34 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:34.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:34 smithi187 ceph-mon[33595]: from='osd.6 [v2:172.21.15.187:6816/1708678714,v1:172.21.15.187:6817/1708678714]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:49:34.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:34 smithi187 ceph-mon[33595]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:49:34.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:34 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:34.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:34 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:34.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:34 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:34.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[41183]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:34.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:34.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:34.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:34.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:34.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:34.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[41183]: from='osd.6 [v2:172.21.15.187:6816/1708678714,v1:172.21.15.187:6817/1708678714]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:49:34.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[41183]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:49:34.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:34.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:34.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:34.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[29765]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:34.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:34.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:34.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:34.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:34.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:34.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[29765]: from='osd.6 [v2:172.21.15.187:6816/1708678714,v1:172.21.15.187:6817/1708678714]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:49:34.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[29765]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:49:34.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:34.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:34.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:34 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:35.287 DEBUG:teuthology.orchestra.run.smithi187:osd.6> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.6.service 2024-03-29T04:49:35.290 INFO:tasks.cephadm:Deploying osd.7 on smithi187 with /dev/vg_nvme/lv_1... 2024-03-29T04:49:35.290 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-29T04:49:35.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:35 smithi187 ceph-mon[33595]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-29T04:49:35.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:35 smithi187 ceph-mon[33595]: osdmap e45: 7 total, 6 up, 7 in 2024-03-29T04:49:35.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:35 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:49:35.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:35 smithi187 ceph-mon[33595]: from='osd.6 [v2:172.21.15.187:6816/1708678714,v1:172.21.15.187:6817/1708678714]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:49:35.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:35 smithi187 ceph-mon[33595]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:49:35.879 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 04:49:35 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T04:49:35.779+0000 7f3ff5258700 -1 osd.6 0 waiting for initial osdmap 2024-03-29T04:49:35.879 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 04:49:35 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T04:49:35.785+0000 7f3fef852700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:49:36.064 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:49:36.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:35 smithi181 ceph-mon[29765]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-29T04:49:36.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:35 smithi181 ceph-mon[29765]: osdmap e45: 7 total, 6 up, 7 in 2024-03-29T04:49:36.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:35 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:49:36.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:35 smithi181 ceph-mon[29765]: from='osd.6 [v2:172.21.15.187:6816/1708678714,v1:172.21.15.187:6817/1708678714]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:49:36.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:35 smithi181 ceph-mon[29765]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:49:36.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:35 smithi181 ceph-mon[41183]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-29T04:49:36.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:35 smithi181 ceph-mon[41183]: osdmap e45: 7 total, 6 up, 7 in 2024-03-29T04:49:36.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:35 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:49:36.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:35 smithi181 ceph-mon[41183]: from='osd.6 [v2:172.21.15.187:6816/1708678714,v1:172.21.15.187:6817/1708678714]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:49:36.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:35 smithi181 ceph-mon[41183]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:49:37.128 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:36 smithi187 ceph-mon[33595]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:37.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:36 smithi187 ceph-mon[33595]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-29T04:49:37.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:36 smithi187 ceph-mon[33595]: osdmap e46: 7 total, 6 up, 7 in 2024-03-29T04:49:37.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:36 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:49:37.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:36 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:49:37.198 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:36 smithi181 ceph-mon[41183]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:37.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:36 smithi181 ceph-mon[41183]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-29T04:49:37.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:36 smithi181 ceph-mon[41183]: osdmap e46: 7 total, 6 up, 7 in 2024-03-29T04:49:37.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:36 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:49:37.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:36 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:49:37.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:36 smithi181 ceph-mon[29765]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:37.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:36 smithi181 ceph-mon[29765]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-29T04:49:37.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:36 smithi181 ceph-mon[29765]: osdmap e46: 7 total, 6 up, 7 in 2024-03-29T04:49:37.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:36 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:49:37.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:36 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:49:37.838 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:37 smithi187 ceph-mon[33595]: purged_snaps scrub starts 2024-03-29T04:49:37.838 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:37 smithi187 ceph-mon[33595]: purged_snaps scrub ok 2024-03-29T04:49:37.838 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:37 smithi187 ceph-mon[33595]: osd.6 [v2:172.21.15.187:6816/1708678714,v1:172.21.15.187:6817/1708678714] boot 2024-03-29T04:49:37.839 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:37 smithi187 ceph-mon[33595]: osdmap e47: 7 total, 7 up, 7 in 2024-03-29T04:49:37.839 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:37 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:49:37.839 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:37 smithi187 ceph-mon[33595]: pgmap v151: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:37.839 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:37 smithi187 ceph-mon[33595]: osdmap e48: 7 total, 7 up, 7 in 2024-03-29T04:49:38.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:37 smithi181 ceph-mon[41183]: purged_snaps scrub starts 2024-03-29T04:49:38.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:37 smithi181 ceph-mon[41183]: purged_snaps scrub ok 2024-03-29T04:49:38.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:37 smithi181 ceph-mon[41183]: osd.6 [v2:172.21.15.187:6816/1708678714,v1:172.21.15.187:6817/1708678714] boot 2024-03-29T04:49:38.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:37 smithi181 ceph-mon[41183]: osdmap e47: 7 total, 7 up, 7 in 2024-03-29T04:49:38.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:37 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:49:38.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:37 smithi181 ceph-mon[41183]: pgmap v151: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:38.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:37 smithi181 ceph-mon[41183]: osdmap e48: 7 total, 7 up, 7 in 2024-03-29T04:49:38.101 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:37 smithi181 ceph-mon[29765]: purged_snaps scrub starts 2024-03-29T04:49:38.101 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:37 smithi181 ceph-mon[29765]: purged_snaps scrub ok 2024-03-29T04:49:38.101 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:37 smithi181 ceph-mon[29765]: osd.6 [v2:172.21.15.187:6816/1708678714,v1:172.21.15.187:6817/1708678714] boot 2024-03-29T04:49:38.101 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:37 smithi181 ceph-mon[29765]: osdmap e47: 7 total, 7 up, 7 in 2024-03-29T04:49:38.101 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:37 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:49:38.101 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:37 smithi181 ceph-mon[29765]: pgmap v151: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:49:38.101 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:37 smithi181 ceph-mon[29765]: osdmap e48: 7 total, 7 up, 7 in 2024-03-29T04:49:38.853 INFO:teuthology.orchestra.run.smithi187.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-29T04:49:38.853 INFO:teuthology.orchestra.run.smithi187.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-29T04:49:38.853 INFO:teuthology.orchestra.run.smithi187.stderr: stderr: 10+0 records in 2024-03-29T04:49:38.853 INFO:teuthology.orchestra.run.smithi187.stderr:10+0 records out 2024-03-29T04:49:38.853 INFO:teuthology.orchestra.run.smithi187.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0114652 s, 915 MB/s 2024-03-29T04:49:38.853 INFO:teuthology.orchestra.run.smithi187.stderr:--> Zapping successful for: 2024-03-29T04:49:39.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:39 smithi181 ceph-mon[41183]: osdmap e49: 7 total, 7 up, 7 in 2024-03-29T04:49:39.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:39 smithi181 ceph-mon[29765]: osdmap e49: 7 total, 7 up, 7 in 2024-03-29T04:49:39.710 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:39 smithi187 ceph-mon[33595]: osdmap e49: 7 total, 7 up, 7 in 2024-03-29T04:49:39.738 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi187:vg_nvme/lv_1 2024-03-29T04:49:40.179 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:49:40.691 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:40 smithi187 ceph-mon[33595]: pgmap v154: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:49:40.691 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:40 smithi187 ceph-mon[33595]: osdmap e50: 7 total, 7 up, 7 in 2024-03-29T04:49:40.691 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:40.691 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:40.691 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:49:40.691 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:49:40.691 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:49:40.691 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:40.691 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:40.691 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:40.692 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:40 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:40.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[41183]: pgmap v154: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:49:40.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[41183]: osdmap e50: 7 total, 7 up, 7 in 2024-03-29T04:49:40.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:40.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:40.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:49:40.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:49:40.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:49:40.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:40.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:40.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:40.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:40.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[29765]: pgmap v154: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:49:40.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[29765]: osdmap e50: 7 total, 7 up, 7 in 2024-03-29T04:49:40.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:40.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:40.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:49:40.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:49:40.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:49:40.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:40.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:40.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:40.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:40 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:41.610 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:41 smithi187 ceph-mon[33595]: Detected new or changed devices on smithi187 2024-03-29T04:49:41.610 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:41 smithi187 ceph-mon[33595]: Adjusting osd_memory_target on smithi187 to 5696M 2024-03-29T04:49:41.610 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:41 smithi187 ceph-mon[33595]: osdmap e51: 7 total, 7 up, 7 in 2024-03-29T04:49:41.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:41 smithi181 ceph-mon[41183]: Detected new or changed devices on smithi187 2024-03-29T04:49:41.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:41 smithi181 ceph-mon[41183]: Adjusting osd_memory_target on smithi187 to 5696M 2024-03-29T04:49:41.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:41 smithi181 ceph-mon[41183]: osdmap e51: 7 total, 7 up, 7 in 2024-03-29T04:49:41.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:41 smithi181 ceph-mon[29765]: Detected new or changed devices on smithi187 2024-03-29T04:49:41.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:41 smithi181 ceph-mon[29765]: Adjusting osd_memory_target on smithi187 to 5696M 2024-03-29T04:49:41.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:41 smithi181 ceph-mon[29765]: osdmap e51: 7 total, 7 up, 7 in 2024-03-29T04:49:42.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:42 smithi181 ceph-mon[41183]: pgmap v157: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:49:42.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:42 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:49:42.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:42 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:49:42.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:42 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:42.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:42 smithi181 ceph-mon[29765]: pgmap v157: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:49:42.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:42 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:49:42.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:42 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:49:42.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:42 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:42.725 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:42 smithi187 ceph-mon[33595]: pgmap v157: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:49:42.725 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:42 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:49:42.725 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:42 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:49:42.725 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:42 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:43.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:43 smithi181 ceph-mon[29765]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:49:43.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:43 smithi181 ceph-mon[41183]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:49:43.757 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:43 smithi187 ceph-mon[33595]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi187:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:49:44.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:44 smithi181 ceph-mon[41183]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-03-29T04:49:44.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:44 smithi181 ceph-mon[29765]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-03-29T04:49:44.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:44 smithi187 ceph-mon[33595]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-03-29T04:49:45.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:45 smithi181 ceph-mon[41183]: from='client.? 172.21.15.187:0/2177241599' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e2520cb-e4ba-4b2a-9206-8a250e7a373f"}]: dispatch 2024-03-29T04:49:45.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:45 smithi181 ceph-mon[41183]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e2520cb-e4ba-4b2a-9206-8a250e7a373f"}]: dispatch 2024-03-29T04:49:45.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:45 smithi181 ceph-mon[41183]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9e2520cb-e4ba-4b2a-9206-8a250e7a373f"}]': finished 2024-03-29T04:49:45.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:45 smithi181 ceph-mon[41183]: osdmap e52: 8 total, 7 up, 8 in 2024-03-29T04:49:45.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:45 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:49:45.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:45 smithi181 ceph-mon[41183]: from='client.? 172.21.15.187:0/655980539' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:49:45.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:45 smithi181 ceph-mon[29765]: from='client.? 172.21.15.187:0/2177241599' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e2520cb-e4ba-4b2a-9206-8a250e7a373f"}]: dispatch 2024-03-29T04:49:45.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:45 smithi181 ceph-mon[29765]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e2520cb-e4ba-4b2a-9206-8a250e7a373f"}]: dispatch 2024-03-29T04:49:45.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:45 smithi181 ceph-mon[29765]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9e2520cb-e4ba-4b2a-9206-8a250e7a373f"}]': finished 2024-03-29T04:49:45.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:45 smithi181 ceph-mon[29765]: osdmap e52: 8 total, 7 up, 8 in 2024-03-29T04:49:45.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:45 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:49:45.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:45 smithi181 ceph-mon[29765]: from='client.? 172.21.15.187:0/655980539' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:49:45.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:45 smithi187 ceph-mon[33595]: from='client.? 172.21.15.187:0/2177241599' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e2520cb-e4ba-4b2a-9206-8a250e7a373f"}]: dispatch 2024-03-29T04:49:45.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:45 smithi187 ceph-mon[33595]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9e2520cb-e4ba-4b2a-9206-8a250e7a373f"}]: dispatch 2024-03-29T04:49:45.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:45 smithi187 ceph-mon[33595]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9e2520cb-e4ba-4b2a-9206-8a250e7a373f"}]': finished 2024-03-29T04:49:45.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:45 smithi187 ceph-mon[33595]: osdmap e52: 8 total, 7 up, 8 in 2024-03-29T04:49:45.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:45 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:49:45.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:45 smithi187 ceph-mon[33595]: from='client.? 172.21.15.187:0/655980539' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:49:46.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:46 smithi187 ceph-mon[33595]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:49:46.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:46 smithi181 ceph-mon[29765]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:49:46.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:46 smithi181 ceph-mon[41183]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:49:48.781 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:48 smithi187 ceph-mon[33595]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:49:48.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:48 smithi181 ceph-mon[29765]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:49:48.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:48 smithi181 ceph-mon[41183]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:49:50.698 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:50 smithi181 ceph-mon[29765]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-29T04:49:50.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:50 smithi181 ceph-mon[41183]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-29T04:49:50.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:50 smithi187 ceph-mon[33595]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-29T04:49:51.878 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:51 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:49:51.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:51 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:51.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:51 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:49:51.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:51 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:51.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:51 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:49:51.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:51 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:52.843 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:52 smithi187 ceph-mon[33595]: Deploying daemon osd.7 on smithi187 2024-03-29T04:49:52.843 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:52 smithi187 ceph-mon[33595]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:49:52.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:52 smithi181 ceph-mon[41183]: Deploying daemon osd.7 on smithi187 2024-03-29T04:49:52.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:52 smithi181 ceph-mon[41183]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:49:52.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:52 smithi181 ceph-mon[29765]: Deploying daemon osd.7 on smithi187 2024-03-29T04:49:52.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:52 smithi181 ceph-mon[29765]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:49:54.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:54 smithi187 ceph-mon[33595]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:49:54.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:54 smithi181 ceph-mon[29765]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:49:54.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:54 smithi181 ceph-mon[41183]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:49:56.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:56 smithi187 ceph-mon[33595]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:49:56.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:56 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:56.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:56 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:56.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:56 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:56.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:56 smithi181 ceph-mon[29765]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:49:56.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:56 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:56.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:56 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:56.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:56 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:56.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:56 smithi181 ceph-mon[41183]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:49:56.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:56 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:56.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:56 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:49:56.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:56 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:58.729 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:49:58 smithi187 ceph-mon[33595]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:49:58.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:49:58 smithi181 ceph-mon[29765]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:49:58.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:49:58 smithi181 ceph-mon[41183]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:49:59.844 INFO:teuthology.orchestra.run.smithi187.stdout:Created osd(s) 7 on host 'smithi187' 2024-03-29T04:50:00.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:00 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:00.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:00 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:00.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:00 smithi187 ceph-mon[33595]: from='osd.7 [v2:172.21.15.187:6824/2139604103,v1:172.21.15.187:6825/2139604103]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:50:00.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:00 smithi187 ceph-mon[33595]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:50:00.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:00 smithi187 ceph-mon[33595]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:50:00.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:00 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:00.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:00 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:00.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:00 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:00.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:00 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:00.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:00 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:00.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:00 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:00.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:00 smithi187 ceph-mon[33595]: overall HEALTH_OK 2024-03-29T04:50:00.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:00.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:00.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[29765]: from='osd.7 [v2:172.21.15.187:6824/2139604103,v1:172.21.15.187:6825/2139604103]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:50:00.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[29765]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:50:00.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[29765]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:50:00.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:00.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:00.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:00.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:00.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:00.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:00.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[29765]: overall HEALTH_OK 2024-03-29T04:50:00.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:00.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:00.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[41183]: from='osd.7 [v2:172.21.15.187:6824/2139604103,v1:172.21.15.187:6825/2139604103]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:50:00.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[41183]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:50:00.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[41183]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:50:00.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:00.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:00.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:00.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:00.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:00.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:00.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:00 smithi181 ceph-mon[41183]: overall HEALTH_OK 2024-03-29T04:50:00.451 DEBUG:teuthology.orchestra.run.smithi187:osd.7> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.7.service 2024-03-29T04:50:00.453 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-03-29T04:50:00.454 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd stat -f json 2024-03-29T04:50:00.719 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:01.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:01 smithi187 ceph-mon[33595]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-29T04:50:01.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:01 smithi187 ceph-mon[33595]: osdmap e53: 8 total, 7 up, 8 in 2024-03-29T04:50:01.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:01 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:50:01.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:01 smithi187 ceph-mon[33595]: from='osd.7 [v2:172.21.15.187:6824/2139604103,v1:172.21.15.187:6825/2139604103]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:50:01.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:01 smithi187 ceph-mon[33595]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:50:01.379 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 04:50:01 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T04:50:01.145+0000 7ff754dc0700 -1 osd.7 0 waiting for initial osdmap 2024-03-29T04:50:01.379 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 04:50:01 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T04:50:01.152+0000 7ff751bbf700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:50:01.448 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:01 smithi181 ceph-mon[29765]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-29T04:50:01.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:01 smithi181 ceph-mon[29765]: osdmap e53: 8 total, 7 up, 8 in 2024-03-29T04:50:01.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:01 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:50:01.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:01 smithi181 ceph-mon[29765]: from='osd.7 [v2:172.21.15.187:6824/2139604103,v1:172.21.15.187:6825/2139604103]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:50:01.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:01 smithi181 ceph-mon[29765]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:50:01.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:01 smithi181 ceph-mon[41183]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-29T04:50:01.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:01 smithi181 ceph-mon[41183]: osdmap e53: 8 total, 7 up, 8 in 2024-03-29T04:50:01.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:01 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:50:01.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:01 smithi181 ceph-mon[41183]: from='osd.7 [v2:172.21.15.187:6824/2139604103,v1:172.21.15.187:6825/2139604103]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:50:01.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:01 smithi181 ceph-mon[41183]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T04:50:02.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:02 smithi181 ceph-mon[29765]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-29T04:50:02.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:02 smithi181 ceph-mon[29765]: osdmap e54: 8 total, 7 up, 8 in 2024-03-29T04:50:02.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:02 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:50:02.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:02 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:50:02.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:02 smithi181 ceph-mon[29765]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:50:02.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:02 smithi181 ceph-mon[41183]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-29T04:50:02.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:02 smithi181 ceph-mon[41183]: osdmap e54: 8 total, 7 up, 8 in 2024-03-29T04:50:02.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:02 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:50:02.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:02 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:50:02.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:02 smithi181 ceph-mon[41183]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:50:02.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:02 smithi187 ceph-mon[33595]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]': finished 2024-03-29T04:50:02.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:02 smithi187 ceph-mon[33595]: osdmap e54: 8 total, 7 up, 8 in 2024-03-29T04:50:02.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:02 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:50:02.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:02 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:50:02.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:02 smithi187 ceph-mon[33595]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:50:02.640 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:50:03.313 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:03 smithi181 ceph-mon[29765]: purged_snaps scrub starts 2024-03-29T04:50:03.313 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:03 smithi181 ceph-mon[29765]: purged_snaps scrub ok 2024-03-29T04:50:03.313 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:03 smithi181 ceph-mon[29765]: osd.7 [v2:172.21.15.187:6824/2139604103,v1:172.21.15.187:6825/2139604103] boot 2024-03-29T04:50:03.314 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:03 smithi181 ceph-mon[29765]: osdmap e55: 8 total, 8 up, 8 in 2024-03-29T04:50:03.314 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:03 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:50:03.314 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:03 smithi181 ceph-mon[29765]: osdmap e56: 8 total, 8 up, 8 in 2024-03-29T04:50:03.314 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:03 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/4285973560' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-29T04:50:03.314 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:03 smithi181 ceph-mon[41183]: purged_snaps scrub starts 2024-03-29T04:50:03.314 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:03 smithi181 ceph-mon[41183]: purged_snaps scrub ok 2024-03-29T04:50:03.314 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:03 smithi181 ceph-mon[41183]: osd.7 [v2:172.21.15.187:6824/2139604103,v1:172.21.15.187:6825/2139604103] boot 2024-03-29T04:50:03.314 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:03 smithi181 ceph-mon[41183]: osdmap e55: 8 total, 8 up, 8 in 2024-03-29T04:50:03.314 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:03 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:50:03.314 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:03 smithi181 ceph-mon[41183]: osdmap e56: 8 total, 8 up, 8 in 2024-03-29T04:50:03.314 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:03 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/4285973560' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-29T04:50:03.341 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":56,"num_osds":8,"num_up_osds":8,"osd_up_since":1711687802,"num_in_osds":8,"osd_in_since":1711687784,"num_remapped_pgs":1} 2024-03-29T04:50:03.341 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:50:03.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:03 smithi187 ceph-mon[33595]: purged_snaps scrub starts 2024-03-29T04:50:03.378 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:03 smithi187 ceph-mon[33595]: purged_snaps scrub ok 2024-03-29T04:50:03.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:03 smithi187 ceph-mon[33595]: osd.7 [v2:172.21.15.187:6824/2139604103,v1:172.21.15.187:6825/2139604103] boot 2024-03-29T04:50:03.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:03 smithi187 ceph-mon[33595]: osdmap e55: 8 total, 8 up, 8 in 2024-03-29T04:50:03.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:03 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:50:03.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:03 smithi187 ceph-mon[33595]: osdmap e56: 8 total, 8 up, 8 in 2024-03-29T04:50:03.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:03 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/4285973560' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-29T04:50:03.608 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:04.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[41183]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:04.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[41183]: osdmap e57: 8 total, 8 up, 8 in 2024-03-29T04:50:04.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[41183]: Detected new or changed devices on smithi187 2024-03-29T04:50:04.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:04.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:04.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:04.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:04.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:04.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:04.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[41183]: Adjusting osd_memory_target on smithi187 to 4272M 2024-03-29T04:50:04.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:04.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:04.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:04.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:04.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[29765]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:04.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[29765]: osdmap e57: 8 total, 8 up, 8 in 2024-03-29T04:50:04.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[29765]: Detected new or changed devices on smithi187 2024-03-29T04:50:04.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:04.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:04.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:04.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:04.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:04.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:04.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[29765]: Adjusting osd_memory_target on smithi187 to 4272M 2024-03-29T04:50:04.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:04.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:04.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:04.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:04 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:04.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:04 smithi187 ceph-mon[33595]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:04.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:04 smithi187 ceph-mon[33595]: osdmap e57: 8 total, 8 up, 8 in 2024-03-29T04:50:04.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:04 smithi187 ceph-mon[33595]: Detected new or changed devices on smithi187 2024-03-29T04:50:04.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:04.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:04.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:04.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:04.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:04.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:04.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:04 smithi187 ceph-mon[33595]: Adjusting osd_memory_target on smithi187 to 4272M 2024-03-29T04:50:04.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:04.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:04.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:04.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:04 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:05.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:05 smithi181 ceph-mon[29765]: osdmap e58: 8 total, 8 up, 8 in 2024-03-29T04:50:05.429 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:05 smithi181 ceph-mon[41183]: osdmap e58: 8 total, 8 up, 8 in 2024-03-29T04:50:05.455 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:50:05.456 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":59,"fsid":"dfcecab8-ed86-11ee-95d0-87774f69a715","created":"2024-03-29T04:44:49.014354+0000","modified":"2024-03-29T04:50:05.419351+0000","last_up_change":"2024-03-29T04:50:02.141186+0000","last_in_change":"2024-03-29T04:49:44.602034+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:48:01.551395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"26991441-a6cb-4694-aec3-0f793de6d551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6803","nonce":135202285}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6805","nonce":135202285}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6809","nonce":135202285}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6807","nonce":135202285}]},"public_addr":"172.21.15.181:6803/135202285","cluster_addr":"172.21.15.181:6805/135202285","heartbeat_back_addr":"172.21.15.181:6809/135202285","heartbeat_front_addr":"172.21.15.181:6807/135202285","state":["exists","up"]},{"osd":1,"uuid":"04b645ce-4eb8-43bc-9f28-e3be932f4b1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6811","nonce":3021678652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6813","nonce":3021678652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6817","nonce":3021678652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6815","nonce":3021678652}]},"public_addr":"172.21.15.181:6811/3021678652","cluster_addr":"172.21.15.181:6813/3021678652","heartbeat_back_addr":"172.21.15.181:6817/3021678652","heartbeat_front_addr":"172.21.15.181:6815/3021678652","state":["exists","up"]},{"osd":2,"uuid":"f72d641c-5d86-4d3a-bf90-d99c1c46a805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6819","nonce":352673207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6821","nonce":352673207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6825","nonce":352673207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6823","nonce":352673207}]},"public_addr":"172.21.15.181:6819/352673207","cluster_addr":"172.21.15.181:6821/352673207","heartbeat_back_addr":"172.21.15.181:6825/352673207","heartbeat_front_addr":"172.21.15.181:6823/352673207","state":["exists","up"]},{"osd":3,"uuid":"63ebbc1a-43b5-4990-87aa-f8f0ff975830","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6827","nonce":1785806254}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6829","nonce":1785806254}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6832","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6833","nonce":1785806254}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6831","nonce":1785806254}]},"public_addr":"172.21.15.181:6827/1785806254","cluster_addr":"172.21.15.181:6829/1785806254","heartbeat_back_addr":"172.21.15.181:6833/1785806254","heartbeat_front_addr":"172.21.15.181:6831/1785806254","state":["exists","up"]},{"osd":4,"uuid":"0a71f228-2cc8-4b83-b86d-62a2854a8533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6800","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6801","nonce":3700510703}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6802","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6803","nonce":3700510703}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6806","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6807","nonce":3700510703}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6804","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6805","nonce":3700510703}]},"public_addr":"172.21.15.187:6801/3700510703","cluster_addr":"172.21.15.187:6803/3700510703","heartbeat_back_addr":"172.21.15.187:6807/3700510703","heartbeat_front_addr":"172.21.15.187:6805/3700510703","state":["exists","up"]},{"osd":5,"uuid":"847657e4-7b9a-4eee-9df5-8e32d0d6543e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6808","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6809","nonce":989620923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6810","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6811","nonce":989620923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6814","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6815","nonce":989620923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6812","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6813","nonce":989620923}]},"public_addr":"172.21.15.187:6809/989620923","cluster_addr":"172.21.15.187:6811/989620923","heartbeat_back_addr":"172.21.15.187:6815/989620923","heartbeat_front_addr":"172.21.15.187:6813/989620923","state":["exists","up"]},{"osd":6,"uuid":"cead714e-ffc1-44a4-bef5-cfa772070c9f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6816","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6817","nonce":1708678714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6818","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6819","nonce":1708678714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6822","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6823","nonce":1708678714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6820","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6821","nonce":1708678714}]},"public_addr":"172.21.15.187:6817/1708678714","cluster_addr":"172.21.15.187:6819/1708678714","heartbeat_back_addr":"172.21.15.187:6823/1708678714","heartbeat_front_addr":"172.21.15.187:6821/1708678714","state":["exists","up"]},{"osd":7,"uuid":"9e2520cb-e4ba-4b2a-9206-8a250e7a373f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6824","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6825","nonce":2139604103}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6826","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6827","nonce":2139604103}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6830","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6831","nonce":2139604103}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6828","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6829","nonce":2139604103}]},"public_addr":"172.21.15.187:6825/2139604103","cluster_addr":"172.21.15.187:6827/2139604103","heartbeat_back_addr":"172.21.15.187:6831/2139604103","heartbeat_front_addr":"172.21.15.187:6829/2139604103","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:08.364426+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:33.120007+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:57.511900+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:48:22.235902+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:48:45.159568+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:49:10.475329+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:49:34.890974+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:50:00.243723+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.181:0/114113941":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/2540916225":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/2863027222":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/1768629494":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6800/2863027222":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/2056930738":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/4052605721":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/1867140711":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6800/2540916225":"2024-03-30T04:45:49.234812+0000","172.21.15.181:0/196688700":"2024-03-30T04:45:25.467994+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:50:05.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:05 smithi187 ceph-mon[33595]: osdmap e58: 8 total, 8 up, 8 in 2024-03-29T04:50:06.187 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-29T04:48:01.551395+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-03-29T04:50:06.188 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd pool get .mgr pg_num 2024-03-29T04:50:06.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:06 smithi181 ceph-mon[29765]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:06.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:06 smithi181 ceph-mon[29765]: osdmap e59: 8 total, 8 up, 8 in 2024-03-29T04:50:06.428 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:06 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2551727523' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:50:06.437 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:06.690 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:06 smithi181 ceph-mon[41183]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:06.690 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:06 smithi181 ceph-mon[41183]: osdmap e59: 8 total, 8 up, 8 in 2024-03-29T04:50:06.690 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:06 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2551727523' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:50:06.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:06 smithi187 ceph-mon[33595]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:06.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:06 smithi187 ceph-mon[33595]: osdmap e59: 8 total, 8 up, 8 in 2024-03-29T04:50:06.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:06 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2551727523' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:50:08.339 INFO:teuthology.orchestra.run.smithi181.stdout:pg_num: 1 2024-03-29T04:50:08.654 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:08 smithi181 ceph-mon[29765]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s, 0 objects/s recovering 2024-03-29T04:50:08.654 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:08 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2829974681' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-29T04:50:08.655 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:08 smithi181 ceph-mon[41183]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s, 0 objects/s recovering 2024-03-29T04:50:08.655 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:08 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2829974681' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-29T04:50:08.859 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:08 smithi187 ceph-mon[33595]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 118 KiB/s, 0 objects/s recovering 2024-03-29T04:50:08.859 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:08 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2829974681' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-29T04:50:08.962 INFO:tasks.cephadm:Adding prometheus.a on smithi187 2024-03-29T04:50:08.962 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch apply prometheus '1;smithi187=a' 2024-03-29T04:50:09.222 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:50:10.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:10 smithi187 ceph-mon[33595]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:50:10.698 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:10 smithi181 ceph-mon[41183]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:50:10.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:10 smithi181 ceph-mon[29765]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:50:11.055 INFO:teuthology.orchestra.run.smithi187.stdout:Scheduled prometheus update... 2024-03-29T04:50:11.697 DEBUG:teuthology.orchestra.run.smithi187:prometheus.a> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@prometheus.a.service 2024-03-29T04:50:11.700 INFO:tasks.cephadm:Adding node-exporter.a on smithi181 2024-03-29T04:50:11.700 INFO:tasks.cephadm:Adding node-exporter.b on smithi187 2024-03-29T04:50:11.700 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch apply node-exporter '2;smithi181=a;smithi187=b' 2024-03-29T04:50:12.013 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:50:12.315 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:12 smithi187 ceph-mon[33595]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi187=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:50:12.315 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:12 smithi187 ceph-mon[33595]: Saving service prometheus spec with placement smithi187=a;count:1 2024-03-29T04:50:12.315 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:12 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:12.315 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:12 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:12.315 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:12 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:12.315 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:12 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:12.316 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:12 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:12.316 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:12 smithi187 ceph-mon[33595]: Deploying daemon prometheus.a on smithi187 2024-03-29T04:50:12.316 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:12 smithi187 ceph-mon[33595]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:50:12.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[29765]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi187=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:50:12.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[29765]: Saving service prometheus spec with placement smithi187=a;count:1 2024-03-29T04:50:12.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:12.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:12.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:12.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:12.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:12.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[29765]: Deploying daemon prometheus.a on smithi187 2024-03-29T04:50:12.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[29765]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:50:12.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[41183]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi187=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:50:12.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[41183]: Saving service prometheus spec with placement smithi187=a;count:1 2024-03-29T04:50:12.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:12.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:12.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:12.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:12.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:12.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[41183]: Deploying daemon prometheus.a on smithi187 2024-03-29T04:50:12.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:12 smithi181 ceph-mon[41183]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:50:14.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:14 smithi187 ceph-mon[33595]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-29T04:50:14.437 INFO:teuthology.orchestra.run.smithi187.stdout:Scheduled node-exporter update... 2024-03-29T04:50:14.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:14 smithi181 ceph-mon[41183]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-29T04:50:14.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:14 smithi181 ceph-mon[29765]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-29T04:50:15.464 DEBUG:teuthology.orchestra.run.smithi181:node-exporter.a> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@node-exporter.a.service 2024-03-29T04:50:15.467 DEBUG:teuthology.orchestra.run.smithi187:node-exporter.b> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@node-exporter.b.service 2024-03-29T04:50:15.469 INFO:tasks.cephadm:Adding alertmanager.a on smithi181 2024-03-29T04:50:15.469 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch apply alertmanager '1;smithi181=a' 2024-03-29T04:50:15.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:15 smithi181 ceph-mon[41183]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi181=a;smithi187=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:50:15.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:15 smithi181 ceph-mon[41183]: Saving service node-exporter spec with placement smithi181=a;smithi187=b;count:2 2024-03-29T04:50:15.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:15 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:15.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:15 smithi181 ceph-mon[29765]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi181=a;smithi187=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:50:15.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:15 smithi181 ceph-mon[29765]: Saving service node-exporter spec with placement smithi181=a;smithi187=b;count:2 2024-03-29T04:50:15.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:15 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:15.777 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:50:15.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:15 smithi187 ceph-mon[33595]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi181=a;smithi187=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:50:15.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:15 smithi187 ceph-mon[33595]: Saving service node-exporter spec with placement smithi181=a;smithi187=b;count:2 2024-03-29T04:50:15.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:15 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:16.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:16 smithi181 ceph-mon[41183]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:50:16.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:16 smithi181 ceph-mon[29765]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:50:16.731 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:16 smithi187 ceph-mon[33595]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:50:18.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:18 smithi187 ceph-mon[33595]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-03-29T04:50:18.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:18 smithi181 ceph-mon[41183]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-03-29T04:50:18.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:18 smithi181 ceph-mon[29765]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2024-03-29T04:50:20.191 INFO:teuthology.orchestra.run.smithi187.stdout:Scheduled alertmanager update... 2024-03-29T04:50:20.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:20 smithi187 ceph-mon[33595]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:20.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:20 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:20.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:20 smithi181 ceph-mon[41183]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:20.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:20 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:20.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:20 smithi181 ceph-mon[29765]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:20.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:20 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:21.586 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:21 smithi187 ceph-mon[33595]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:50:21.586 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:21 smithi187 ceph-mon[33595]: Saving service alertmanager spec with placement smithi181=a;count:1 2024-03-29T04:50:21.613 DEBUG:teuthology.orchestra.run.smithi181:alertmanager.a> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@alertmanager.a.service 2024-03-29T04:50:21.616 INFO:tasks.cephadm:Adding grafana.a on smithi187 2024-03-29T04:50:21.616 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph orch apply grafana '1;smithi187=a' 2024-03-29T04:50:21.642 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:21 smithi181 ceph-mon[41183]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:50:21.642 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:21 smithi181 ceph-mon[41183]: Saving service alertmanager spec with placement smithi181=a;count:1 2024-03-29T04:50:21.642 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:21 smithi181 ceph-mon[29765]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi181=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:50:21.642 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:21 smithi181 ceph-mon[29765]: Saving service alertmanager spec with placement smithi181=a;count:1 2024-03-29T04:50:22.731 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:22 smithi187 ceph-mon[33595]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:22.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:22 smithi181 ceph-mon[29765]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:22.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:22 smithi181 ceph-mon[41183]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:23.642 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:50:24.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:24 smithi187 ceph-mon[33595]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:24.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:24 smithi181 ceph-mon[41183]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:24.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:24 smithi181 ceph-mon[29765]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:25.557 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 podman[63437]: 2024-03-29 04:50:25.369239427 +0000 UTC m=+0.356306925 container create 1be4c994efc472443d58ab87580057935c07836ad701ba4d1ebad66c2a0c3647 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:50:25.821 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 podman[63437]: 2024-03-29 04:50:25.763677814 +0000 UTC m=+0.750745315 container init 1be4c994efc472443d58ab87580057935c07836ad701ba4d1ebad66c2a0c3647 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:50:25.821 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 podman[63437]: 2024-03-29 04:50:25.769611634 +0000 UTC m=+0.756679142 container start 1be4c994efc472443d58ab87580057935c07836ad701ba4d1ebad66c2a0c3647 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:50:25.821 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.791Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-29T04:50:25.822 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.791Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-29T04:50:25.822 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.791Z caller=main.go:570 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi187 (none))" 2024-03-29T04:50:25.822 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.791Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-29T04:50:25.822 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.791Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-29T04:50:25.822 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.793Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-29T04:50:25.822 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.794Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-29T04:50:25.822 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.795Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-29T04:50:25.822 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.795Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-29T04:50:25.822 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.798Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-29T04:50:25.822 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.798Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.15µs 2024-03-29T04:50:25.822 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.798Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-29T04:50:25.823 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.798Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-03-29T04:50:25.823 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.798Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.085µs wal_replay_duration=237.57µs wbl_replay_duration=130ns total_replay_duration=295.715µs 2024-03-29T04:50:25.823 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.799Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-29T04:50:25.823 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.799Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-29T04:50:25.823 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.799Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-29T04:50:25.823 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.818Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.52422ms db_storage=930ns remote_storage=1.565µs web_handler=353ns query_engine=637ns scrape=618.334µs scrape_sd=32.764µs notify=1.144µs notify_sd=2.333µs rules=17.377661ms tracing=7.183µs 2024-03-29T04:50:25.823 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.818Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-29T04:50:25.823 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:50:25.818Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-29T04:50:25.884 INFO:teuthology.orchestra.run.smithi187.stdout:Scheduled grafana update... 2024-03-29T04:50:26.129 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 bash[63437]: 1be4c994efc472443d58ab87580057935c07836ad701ba4d1ebad66c2a0c3647 2024-03-29T04:50:26.129 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:50:25 smithi187 systemd[1]: Started Ceph prometheus.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:50:26.511 DEBUG:teuthology.orchestra.run.smithi187:grafana.a> sudo journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service 2024-03-29T04:50:26.513 INFO:tasks.cephadm:Setting up client nodes... 2024-03-29T04:50:26.514 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-03-29T04:50:26.777 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:26.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:26 smithi187 ceph-mon[33595]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:26.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:26 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:26.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:26 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:26.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:26 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:26.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:26 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:26.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:26 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-29T04:50:26.899 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:26 smithi181 ceph-mon[29765]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:26.899 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:26 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:26.899 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:26 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:26.899 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:26 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:26.899 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:26 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:26.899 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:26 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-29T04:50:26.899 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:26 smithi181 ceph-mon[41183]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:26.899 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:26 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:26.899 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:26 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:26.899 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:26 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:26.899 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:26 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' 2024-03-29T04:50:26.899 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:26 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-29T04:50:27.159 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:27 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ignoring --setuser ceph since I am not root 2024-03-29T04:50:27.159 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:27 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ignoring --setgroup ceph since I am not root 2024-03-29T04:50:27.159 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:27 smithi181 ceph-mgr[30042]: -- 172.21.15.181:0/1052530556 <== mon.0 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561195ae89c0 con 0x561195abec00 2024-03-29T04:50:27.379 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ignoring --setuser ceph since I am not root 2024-03-29T04:50:27.379 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ignoring --setgroup ceph since I am not root 2024-03-29T04:50:27.379 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:27 smithi187 ceph-mgr[35709]: -- 172.21.15.187:0/1693218977 <== mon.1 v2:172.21.15.187:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e8bbc349c0 con 0x55e8bbc0b000 2024-03-29T04:50:27.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:27 smithi187 ceph-mon[33595]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi187=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:50:27.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:27 smithi187 ceph-mon[33595]: Saving service grafana spec with placement smithi187=a;count:1 2024-03-29T04:50:27.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:27 smithi187 ceph-mon[33595]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-29T04:50:27.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:27 smithi187 ceph-mon[33595]: mgrmap e16: y(active, since 4m), standbys: x 2024-03-29T04:50:27.880 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:27.519+0000 7f0e8e1d71c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:50:27.880 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:27.611+0000 7f0e8e1d71c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:50:27.949 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:27 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:27.539+0000 7f74f14901c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:50:27.949 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:27 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:27.631+0000 7f74f14901c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:50:27.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:27 smithi181 ceph-mon[29765]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi187=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:50:27.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:27 smithi181 ceph-mon[29765]: Saving service grafana spec with placement smithi187=a;count:1 2024-03-29T04:50:27.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:27 smithi181 ceph-mon[29765]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-29T04:50:27.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:27 smithi181 ceph-mon[29765]: mgrmap e16: y(active, since 4m), standbys: x 2024-03-29T04:50:27.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:27 smithi181 ceph-mon[41183]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi187=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:50:27.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:27 smithi181 ceph-mon[41183]: Saving service grafana spec with placement smithi187=a;count:1 2024-03-29T04:50:27.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:27 smithi181 ceph-mon[41183]: from='mgr.14152 172.21.15.181:0/269228625' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-29T04:50:27.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:27 smithi181 ceph-mon[41183]: mgrmap e16: y(active, since 4m), standbys: x 2024-03-29T04:50:28.556 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:28.230+0000 7f0e8e1d71c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:50:28.556 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:28.318+0000 7f0e8e1d71c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:50:28.556 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:28.400+0000 7f0e8e1d71c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:50:28.611 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:28.274+0000 7f74f14901c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:50:28.611 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:28.364+0000 7f74f14901c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:50:28.611 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:28.452+0000 7f74f14901c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:50:28.611 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:28.607+0000 7f74f14901c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:50:28.771 INFO:teuthology.orchestra.run.smithi181.stdout:[client.0] 2024-03-29T04:50:28.771 INFO:teuthology.orchestra.run.smithi181.stdout: key = AQCUSAZmreaYLRAAOtkEKi3ZKz5OcInAf1XxLg== 2024-03-29T04:50:28.813 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:28.551+0000 7f0e8e1d71c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:50:28.814 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:28.634+0000 7f0e8e1d71c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:50:28.875 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:28.687+0000 7f74f14901c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:50:28.875 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:28 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1717519366' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:50:28.876 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:28 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1717519366' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:50:28.876 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:28 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1717519366' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:50:28.876 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:28 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1717519366' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:50:29.091 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:28 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1717519366' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:50:29.091 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:28 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1717519366' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:50:29.091 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:28.809+0000 7f0e8e1d71c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:50:29.091 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:29 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:29.000+0000 7f0e8e1d71c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:50:29.091 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:29 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:29.086+0000 7f0e8e1d71c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:50:29.141 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:28.871+0000 7f74f14901c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:50:29.141 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:29 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:29.053+0000 7f74f14901c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:50:29.379 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:29 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:29.180+0000 7f0e8e1d71c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:50:29.379 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:29 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:29.268+0000 7f0e8e1d71c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:50:29.430 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:29 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:29.137+0000 7f74f14901c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:50:29.430 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:29 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:29.232+0000 7f74f14901c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:50:29.431 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:29 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:29.318+0000 7f74f14901c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:50:29.459 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-03-29T04:50:29.459 DEBUG:teuthology.orchestra.run.smithi181:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-03-29T04:50:29.459 DEBUG:teuthology.orchestra.run.smithi181:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-03-29T04:50:29.503 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-03-29T04:50:29.819 INFO:teuthology.orchestra.run.smithi187.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:50:30.629 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:30 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:30.351+0000 7f0e8e1d71c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:50:30.629 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:30 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:30.480+0000 7f0e8e1d71c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:50:30.684 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:30.362+0000 7f74f14901c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:50:30.684 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:30.489+0000 7f74f14901c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:50:30.909 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:30 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:30.675+0000 7f0e8e1d71c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:50:30.909 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:30 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:30.906+0000 7f0e8e1d71c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:50:30.949 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:30.680+0000 7f74f14901c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:50:30.949 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:30.909+0000 7f74f14901c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:50:31.264 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:31.081+0000 7f0e8e1d71c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:50:31.449 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:31.084+0000 7f74f14901c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:50:31.449 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:31.262+0000 7f74f14901c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:50:31.524 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:31.261+0000 7f0e8e1d71c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:50:31.785 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:31.521+0000 7f0e8e1d71c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:50:31.785 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:31.607+0000 7f0e8e1d71c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:50:31.949 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:31.524+0000 7f74f14901c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:50:31.949 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:31.610+0000 7f74f14901c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:50:32.129 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:32 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:32.011+0000 7f0e8e1d71c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:50:32.449 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:32 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:31.999+0000 7f74f14901c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:50:32.990 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:32 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:32.826+0000 7f0e8e1d71c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:50:33.068 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:32 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:32.772+0000 7f74f14901c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:50:33.302 INFO:teuthology.orchestra.run.smithi187.stdout:[client.1] 2024-03-29T04:50:33.302 INFO:teuthology.orchestra.run.smithi187.stdout: key = AQCZSAZmT3SdERAAjiQwleNgdP/80pPsdqN9Pw== 2024-03-29T04:50:33.302 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:33 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:33.130+0000 7f0e8e1d71c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:50:33.351 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:33 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:33.064+0000 7f74f14901c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:50:33.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:33 smithi187 ceph-mon[33595]: from='client.? 172.21.15.187:0/2435703062' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:50:33.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:33 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:50:33.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:33 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:50:33.629 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:33 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:33.463+0000 7f0e8e1d71c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:50:33.632 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:33 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:33.381+0000 7f74f14901c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:50:33.632 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:33 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:33.573+0000 7f74f14901c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:50:33.632 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:33 smithi181 ceph-mon[41183]: from='client.? 172.21.15.187:0/2435703062' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:50:33.632 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:33 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:50:33.632 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:33 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:50:33.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:33 smithi181 ceph-mon[29765]: from='client.? 172.21.15.187:0/2435703062' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:50:33.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:33 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:50:33.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:33 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:50:33.949 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:33 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:50:33] ENGINE Bus STARTING 2024-03-29T04:50:33.949 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:33 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: CherryPy Checker: 2024-03-29T04:50:33.949 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:33 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: The Application mounted at '' has an empty config. 2024-03-29T04:50:33.949 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:33 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:50:33.949 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:33 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:50:33] ENGINE Serving on http://:::9283 2024-03-29T04:50:33.949 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:33 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:50:33] ENGINE Bus STARTED 2024-03-29T04:50:34.086 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:33 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:33.659+0000 7f0e8e1d71c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:50:34.086 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:33 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: [29/Mar/2024:04:50:33] ENGINE Bus STARTING 2024-03-29T04:50:34.086 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:33 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: CherryPy Checker: 2024-03-29T04:50:34.086 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:33 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: The Application mounted at '' has an empty config. 2024-03-29T04:50:34.086 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:33 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: 2024-03-29T04:50:34.087 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:33 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: [29/Mar/2024:04:50:33] ENGINE Serving on http://:::9283 2024-03-29T04:50:34.087 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:33 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: [29/Mar/2024:04:50:33] ENGINE Bus STARTED 2024-03-29T04:50:34.113 DEBUG:teuthology.orchestra.run.smithi187:> set -ex 2024-03-29T04:50:34.113 DEBUG:teuthology.orchestra.run.smithi187:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-03-29T04:50:34.113 DEBUG:teuthology.orchestra.run.smithi187:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-03-29T04:50:34.158 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-03-29T04:50:34.158 DEBUG:teuthology.orchestra.run.smithi181:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-03-29T04:50:34.441 INFO:teuthology.misc.health.smithi181.stdout: 2024-03-29T04:50:34.441 INFO:teuthology.misc.health.smithi181.stdout:{"epoch":60,"fsid":"dfcecab8-ed86-11ee-95d0-87774f69a715","created":"2024-03-29T04:44:49.014354+0000","modified":"2024-03-29T04:50:33.576122+0000","last_up_change":"2024-03-29T04:50:02.141186+0000","last_in_change":"2024-03-29T04:49:44.602034+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:48:01.551395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"26991441-a6cb-4694-aec3-0f793de6d551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6803","nonce":135202285}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6805","nonce":135202285}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6809","nonce":135202285}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6807","nonce":135202285}]},"public_addr":"172.21.15.181:6803/135202285","cluster_addr":"172.21.15.181:6805/135202285","heartbeat_back_addr":"172.21.15.181:6809/135202285","heartbeat_front_addr":"172.21.15.181:6807/135202285","state":["exists","up"]},{"osd":1,"uuid":"04b645ce-4eb8-43bc-9f28-e3be932f4b1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6811","nonce":3021678652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6813","nonce":3021678652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6817","nonce":3021678652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6815","nonce":3021678652}]},"public_addr":"172.21.15.181:6811/3021678652","cluster_addr":"172.21.15.181:6813/3021678652","heartbeat_back_addr":"172.21.15.181:6817/3021678652","heartbeat_front_addr":"172.21.15.181:6815/3021678652","state":["exists","up"]},{"osd":2,"uuid":"f72d641c-5d86-4d3a-bf90-d99c1c46a805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6819","nonce":352673207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6821","nonce":352673207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6825","nonce":352673207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6823","nonce":352673207}]},"public_addr":"172.21.15.181:6819/352673207","cluster_addr":"172.21.15.181:6821/352673207","heartbeat_back_addr":"172.21.15.181:6825/352673207","heartbeat_front_addr":"172.21.15.181:6823/352673207","state":["exists","up"]},{"osd":3,"uuid":"63ebbc1a-43b5-4990-87aa-f8f0ff975830","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6827","nonce":1785806254}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6829","nonce":1785806254}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6832","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6833","nonce":1785806254}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6831","nonce":1785806254}]},"public_addr":"172.21.15.181:6827/1785806254","cluster_addr":"172.21.15.181:6829/1785806254","heartbeat_back_addr":"172.21.15.181:6833/1785806254","heartbeat_front_addr":"172.21.15.181:6831/1785806254","state":["exists","up"]},{"osd":4,"uuid":"0a71f228-2cc8-4b83-b86d-62a2854a8533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6800","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6801","nonce":3700510703}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6802","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6803","nonce":3700510703}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6806","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6807","nonce":3700510703}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6804","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6805","nonce":3700510703}]},"public_addr":"172.21.15.187:6801/3700510703","cluster_addr":"172.21.15.187:6803/3700510703","heartbeat_back_addr":"172.21.15.187:6807/3700510703","heartbeat_front_addr":"172.21.15.187:6805/3700510703","state":["exists","up"]},{"osd":5,"uuid":"847657e4-7b9a-4eee-9df5-8e32d0d6543e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6808","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6809","nonce":989620923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6810","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6811","nonce":989620923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6814","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6815","nonce":989620923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6812","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6813","nonce":989620923}]},"public_addr":"172.21.15.187:6809/989620923","cluster_addr":"172.21.15.187:6811/989620923","heartbeat_back_addr":"172.21.15.187:6815/989620923","heartbeat_front_addr":"172.21.15.187:6813/989620923","state":["exists","up"]},{"osd":6,"uuid":"cead714e-ffc1-44a4-bef5-cfa772070c9f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6816","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6817","nonce":1708678714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6818","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6819","nonce":1708678714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6822","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6823","nonce":1708678714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6820","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6821","nonce":1708678714}]},"public_addr":"172.21.15.187:6817/1708678714","cluster_addr":"172.21.15.187:6819/1708678714","heartbeat_back_addr":"172.21.15.187:6823/1708678714","heartbeat_front_addr":"172.21.15.187:6821/1708678714","state":["exists","up"]},{"osd":7,"uuid":"9e2520cb-e4ba-4b2a-9206-8a250e7a373f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6824","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6825","nonce":2139604103}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6826","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6827","nonce":2139604103}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6830","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6831","nonce":2139604103}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6828","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6829","nonce":2139604103}]},"public_addr":"172.21.15.187:6825/2139604103","cluster_addr":"172.21.15.187:6827/2139604103","heartbeat_back_addr":"172.21.15.187:6831/2139604103","heartbeat_front_addr":"172.21.15.187:6829/2139604103","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:08.364426+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:33.120007+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:57.511900+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:48:22.235902+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:48:45.159568+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:49:10.475329+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:49:34.890974+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:50:00.243723+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.181:6800/3044637144":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/4244016576":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/3491810134":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/3153434425":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/114113941":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/2540916225":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/2863027222":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/19173868":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/1768629494":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6800/2863027222":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/2056930738":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/4052605721":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/1867140711":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6800/2540916225":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/3044637144":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/196688700":"2024-03-30T04:45:25.467994+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:50:34.453 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-03-29T04:50:34.453 INFO:tasks.cephadm:Creating RBD pool 2024-03-29T04:50:34.454 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-03-29T04:50:34.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: Active manager daemon y restarted 2024-03-29T04:50:34.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: Activating manager daemon y 2024-03-29T04:50:34.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: osdmap e60: 8 total, 8 up, 8 in 2024-03-29T04:50:34.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: mgrmap e17: y(active, starting, since 0.0064797s), standbys: x 2024-03-29T04:50:34.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:50:34.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:50:34.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:50:34.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:50:34.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:50:34.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:50:34.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:50:34.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:50:34.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:50:34.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:50:34.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:50:34.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:50:34.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:50:34.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:50:34.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:50:34.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:50:34.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: Manager daemon y is now available 2024-03-29T04:50:34.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:34.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:34.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:34.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:34.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:50:34.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:50:34.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: Standby manager daemon x restarted 2024-03-29T04:50:34.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: Standby manager daemon x started 2024-03-29T04:50:34.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/2376784091' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:50:34.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/2376784091' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:50:34.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/2376784091' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:50:34.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/2376784091' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:50:34.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: [29/Mar/2024:04:50:33] ENGINE Bus STARTING 2024-03-29T04:50:34.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: [29/Mar/2024:04:50:34] ENGINE Serving on https://172.21.15.181:7150 2024-03-29T04:50:34.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: [29/Mar/2024:04:50:34] ENGINE Bus STARTED 2024-03-29T04:50:34.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:34.632 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: Active manager daemon y restarted 2024-03-29T04:50:34.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: Activating manager daemon y 2024-03-29T04:50:34.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: osdmap e60: 8 total, 8 up, 8 in 2024-03-29T04:50:34.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: mgrmap e17: y(active, starting, since 0.0064797s), standbys: x 2024-03-29T04:50:34.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:50:34.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:50:34.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:50:34.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:50:34.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:50:34.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:50:34.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:50:34.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:50:34.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:50:34.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:50:34.633 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:50:34.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:50:34.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:50:34.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:50:34.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:50:34.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:50:34.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: Manager daemon y is now available 2024-03-29T04:50:34.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:34.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:34.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:34.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:34.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:50:34.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:50:34.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: Standby manager daemon x restarted 2024-03-29T04:50:34.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: Standby manager daemon x started 2024-03-29T04:50:34.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: Active manager daemon y restarted 2024-03-29T04:50:34.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: Activating manager daemon y 2024-03-29T04:50:34.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: osdmap e60: 8 total, 8 up, 8 in 2024-03-29T04:50:34.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: mgrmap e17: y(active, starting, since 0.0064797s), standbys: x 2024-03-29T04:50:34.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:50:34.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:50:34.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:50:34.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:50:34.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:50:34.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:50:34.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:50:34.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:50:34.636 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:50:34.636 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:50:34.636 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:50:34.636 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:50:34.636 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:50:34.636 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:50:34.636 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:50:34.636 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:50:34.636 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: Manager daemon y is now available 2024-03-29T04:50:34.636 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:34.636 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:34.636 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:34.636 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:34.636 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:50:34.636 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:50:34.636 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: Standby manager daemon x restarted 2024-03-29T04:50:34.637 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: Standby manager daemon x started 2024-03-29T04:50:34.637 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/2376784091' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:50:34.637 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/2376784091' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:50:34.637 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/2376784091' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:50:34.637 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/2376784091' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:50:34.637 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: [29/Mar/2024:04:50:33] ENGINE Bus STARTING 2024-03-29T04:50:34.637 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: [29/Mar/2024:04:50:34] ENGINE Serving on https://172.21.15.181:7150 2024-03-29T04:50:34.637 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: [29/Mar/2024:04:50:34] ENGINE Bus STARTED 2024-03-29T04:50:34.637 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:34.638 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/2376784091' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:50:34.638 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/2376784091' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:50:34.638 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/2376784091' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:50:34.638 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/2376784091' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:50:34.638 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: [29/Mar/2024:04:50:33] ENGINE Bus STARTING 2024-03-29T04:50:34.638 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: [29/Mar/2024:04:50:34] ENGINE Serving on https://172.21.15.181:7150 2024-03-29T04:50:34.638 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: [29/Mar/2024:04:50:34] ENGINE Bus STARTED 2024-03-29T04:50:34.638 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:35.155 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:35.539 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:35 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/137023231' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:50:35.539 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:35 smithi181 ceph-mon[29765]: mgrmap e18: y(active, since 1.0099s), standbys: x 2024-03-29T04:50:35.539 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:35 smithi181 ceph-mon[29765]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:35.540 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:35 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/137023231' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:50:35.540 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:35 smithi181 ceph-mon[41183]: mgrmap e18: y(active, since 1.0099s), standbys: x 2024-03-29T04:50:35.540 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:35 smithi181 ceph-mon[41183]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:35.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:35 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/137023231' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:50:35.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:35 smithi187 ceph-mon[33595]: mgrmap e18: y(active, since 1.0099s), standbys: x 2024-03-29T04:50:35.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:35 smithi187 ceph-mon[33595]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:36.686 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:36 smithi181 ceph-mon[41183]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:36.686 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:36 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:36.686 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:36 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:36.686 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:36 smithi181 ceph-mon[41183]: mgrmap e19: y(active, since 2s), standbys: x 2024-03-29T04:50:36.686 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:36 smithi181 ceph-mon[29765]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:36.687 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:36 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:36.687 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:36 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:36.687 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:36 smithi181 ceph-mon[29765]: mgrmap e19: y(active, since 2s), standbys: x 2024-03-29T04:50:36.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:36 smithi187 ceph-mon[33595]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:36.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:36 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:36.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:36 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:36.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:36 smithi187 ceph-mon[33595]: mgrmap e19: y(active, since 2s), standbys: x 2024-03-29T04:50:38.049 INFO:teuthology.orchestra.run.smithi181.stderr:pool 'rbd' created 2024-03-29T04:50:38.049 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.049 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.049 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:38.049 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:38.049 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:38.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:38.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[29765]: Adjusting osd_memory_target on smithi187 to 4016M 2024-03-29T04:50:38.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2741368039' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-29T04:50:38.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-29T04:50:38.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:38.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:38.050 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.051 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.051 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:38.051 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:38.051 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:38.051 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:38.051 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[41183]: Adjusting osd_memory_target on smithi187 to 4016M 2024-03-29T04:50:38.051 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.051 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.051 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.051 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2741368039' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-29T04:50:38.051 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-29T04:50:38.051 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.051 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.051 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:38.051 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:37 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:38.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:37 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:37 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:37 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:38.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:37 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:38.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:37 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:38.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:37 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:50:38.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:37 smithi187 ceph-mon[33595]: Adjusting osd_memory_target on smithi187 to 4016M 2024-03-29T04:50:38.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:37 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:37 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:37 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:37 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2741368039' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-29T04:50:38.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:37 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-29T04:50:38.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:37 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:37 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:38.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:37 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:38.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:37 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:38.879 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:38 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:50:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:50:38.880 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-03-29T04:50:39.171 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:39.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[41183]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:39.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[41183]: Updating smithi181:/etc/ceph/ceph.conf 2024-03-29T04:50:39.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[41183]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:50:39.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[41183]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:50:39.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[41183]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:50:39.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[41183]: mgrmap e20: y(active, since 4s), standbys: x 2024-03-29T04:50:39.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-03-29T04:50:39.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[41183]: osdmap e61: 8 total, 8 up, 8 in 2024-03-29T04:50:39.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:39.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:39.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:39.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:39.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:39.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[29765]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:39.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[29765]: Updating smithi181:/etc/ceph/ceph.conf 2024-03-29T04:50:39.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[29765]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:50:39.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[29765]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:50:39.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[29765]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:50:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[29765]: mgrmap e20: y(active, since 4s), standbys: x 2024-03-29T04:50:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-03-29T04:50:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[29765]: osdmap e61: 8 total, 8 up, 8 in 2024-03-29T04:50:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:39 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:39.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:39 smithi187 ceph-mon[33595]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:39.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:39 smithi187 ceph-mon[33595]: Updating smithi181:/etc/ceph/ceph.conf 2024-03-29T04:50:39.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:39 smithi187 ceph-mon[33595]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:50:39.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:39 smithi187 ceph-mon[33595]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:50:39.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:39 smithi187 ceph-mon[33595]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:50:39.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:39 smithi187 ceph-mon[33595]: mgrmap e20: y(active, since 4s), standbys: x 2024-03-29T04:50:39.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:39 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-03-29T04:50:39.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:39 smithi187 ceph-mon[33595]: osdmap e61: 8 total, 8 up, 8 in 2024-03-29T04:50:39.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:39 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:39.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:39 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:39.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:39 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:39.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:39 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:39.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:39 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:40.378 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:40 smithi181 ceph-mon[29765]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:50:40.378 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:40 smithi181 ceph-mon[29765]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:50:40.378 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:40 smithi181 ceph-mon[29765]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:50:40.378 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:40 smithi181 ceph-mon[29765]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:50:40.378 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:40 smithi181 ceph-mon[29765]: Deploying daemon alertmanager.a on smithi181 2024-03-29T04:50:40.378 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:40 smithi181 ceph-mon[29765]: osdmap e62: 8 total, 8 up, 8 in 2024-03-29T04:50:40.379 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:40 smithi181 ceph-mon[41183]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:50:40.379 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:40 smithi181 ceph-mon[41183]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:50:40.379 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:40 smithi181 ceph-mon[41183]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:50:40.379 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:40 smithi181 ceph-mon[41183]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:50:40.379 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:40 smithi181 ceph-mon[41183]: Deploying daemon alertmanager.a on smithi181 2024-03-29T04:50:40.379 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:40 smithi181 ceph-mon[41183]: osdmap e62: 8 total, 8 up, 8 in 2024-03-29T04:50:40.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:40 smithi187 ceph-mon[33595]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:50:40.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:40 smithi187 ceph-mon[33595]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:50:40.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:40 smithi187 ceph-mon[33595]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:50:40.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:40 smithi187 ceph-mon[33595]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:50:40.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:40 smithi187 ceph-mon[33595]: Deploying daemon alertmanager.a on smithi181 2024-03-29T04:50:40.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:40 smithi187 ceph-mon[33595]: osdmap e62: 8 total, 8 up, 8 in 2024-03-29T04:50:41.160 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:41 smithi181 ceph-mon[29765]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:41.161 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:41 smithi181 ceph-mon[29765]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:50:41.161 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:41 smithi181 ceph-mon[29765]: osdmap e63: 8 total, 8 up, 8 in 2024-03-29T04:50:41.161 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:41 smithi181 ceph-mon[41183]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:41.161 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:41 smithi181 ceph-mon[41183]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:50:41.161 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:41 smithi181 ceph-mon[41183]: osdmap e63: 8 total, 8 up, 8 in 2024-03-29T04:50:41.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:41 smithi187 ceph-mon[33595]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:41.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:41 smithi187 ceph-mon[33595]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:50:41.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:41 smithi187 ceph-mon[33595]: osdmap e63: 8 total, 8 up, 8 in 2024-03-29T04:50:42.069 INFO:teuthology.orchestra.run.smithi181.stderr:enabled application 'rbd' on pool 'rbd' 2024-03-29T04:50:42.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:42 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3089645537' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:50:42.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:42 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:50:42.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:42 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3089645537' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:50:42.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:42 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:50:42.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:42 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3089645537' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:50:42.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:42 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:50:43.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:43 smithi187 ceph-mon[33595]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-03-29T04:50:43.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:43 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:50:43.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:43 smithi187 ceph-mon[33595]: osdmap e64: 8 total, 8 up, 8 in 2024-03-29T04:50:43.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:43 smithi181 ceph-mon[41183]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-03-29T04:50:43.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:43 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:50:43.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:43 smithi181 ceph-mon[41183]: osdmap e64: 8 total, 8 up, 8 in 2024-03-29T04:50:43.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:43 smithi181 ceph-mon[29765]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-03-29T04:50:43.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:43 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:50:43.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:43 smithi181 ceph-mon[29765]: osdmap e64: 8 total, 8 up, 8 in 2024-03-29T04:50:43.608 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-29T04:50:43.608 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-29T04:50:43.608 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph mgr dump --format=json 2024-03-29T04:50:44.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:44 smithi181 ceph-mon[29765]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 20 op/s 2024-03-29T04:50:44.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:44 smithi181 ceph-mon[29765]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:50:44.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:44 smithi181 ceph-mon[29765]: Cluster is now healthy 2024-03-29T04:50:44.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:44 smithi181 ceph-mon[41183]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 20 op/s 2024-03-29T04:50:44.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:44 smithi181 ceph-mon[41183]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:50:44.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:44 smithi181 ceph-mon[41183]: Cluster is now healthy 2024-03-29T04:50:44.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:44 smithi187 ceph-mon[33595]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 20 op/s 2024-03-29T04:50:44.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:44 smithi187 ceph-mon[33595]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:50:44.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:44 smithi187 ceph-mon[33595]: Cluster is now healthy 2024-03-29T04:50:44.725 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:46.891 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:46 smithi181 ceph-mon[29765]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-03-29T04:50:46.891 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:46 smithi181 ceph-mon[41183]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-03-29T04:50:47.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:46 smithi187 ceph-mon[33595]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 0 B/s wr, 17 op/s 2024-03-29T04:50:47.829 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:47 smithi181 systemd[1]: Starting Ceph alertmanager.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:50:48.606 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:50:48.616 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:48 smithi181 podman[72405]: 2024-03-29 04:50:48.256871033 +0000 UTC m=+0.122465100 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-03-29T04:50:48.616 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:48 smithi181 podman[72405]: 2024-03-29 04:50:48.463475211 +0000 UTC m=+0.329069275 volume create afb1cd572f61bf3de99a440ccb68335689d22ec06e95d6c91bc92b9bd326019b 2024-03-29T04:50:48.616 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:48 smithi181 podman[72405]: 2024-03-29 04:50:48.564152973 +0000 UTC m=+0.429747037 container create 79e961bfe034831b50ec306b60700047b537dd433482668a330da0dc476b3f4c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:50:48.879 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:50:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:50:48.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:48 smithi187 ceph-mon[33595]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-03-29T04:50:48.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:48 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1478675896' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:50:48.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:48 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:48.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:48 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:48.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:48 smithi181 ceph-mon[29765]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-03-29T04:50:48.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:48 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1478675896' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:50:48.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:48 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:48.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:48 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:48.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:48 smithi181 ceph-mon[41183]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-03-29T04:50:48.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:48 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1478675896' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:50:48.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:48 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:48.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:48 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:49.354 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":20,"active_gid":14448,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":3281522413},{"type":"v1","addr":"172.21.15.181:6801","nonce":3281522413}]},"active_addr":"172.21.15.181:6801/3281522413","active_change":"2024-03-29T04:50:33.576252+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24334,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.181:8443/","prometheus":"http://172.21.15.181:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.181:0","nonce":3860889501}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.181:0","nonce":1741596559}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.181:0","nonce":2280393448}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.181:0","nonce":1220271702}]}]} 2024-03-29T04:50:49.358 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-29T04:50:49.358 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-29T04:50:49.358 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:50:49.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:49 smithi181 podman[72405]: 2024-03-29 04:50:49.437160706 +0000 UTC m=+1.302754771 container init 79e961bfe034831b50ec306b60700047b537dd433482668a330da0dc476b3f4c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:50:49.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:49 smithi181 podman[72405]: 2024-03-29 04:50:49.443259946 +0000 UTC m=+1.308854011 container start 79e961bfe034831b50ec306b60700047b537dd433482668a330da0dc476b3f4c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:50:49.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:49 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[72459]: ts=2024-03-29T04:50:49.455Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-03-29T04:50:49.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:49 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[72459]: ts=2024-03-29T04:50:49.455Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-03-29T04:50:49.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:49 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[72459]: ts=2024-03-29T04:50:49.456Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.181 port=9094 2024-03-29T04:50:49.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:49 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[72459]: ts=2024-03-29T04:50:49.457Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-03-29T04:50:49.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:49 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[72459]: ts=2024-03-29T04:50:49.486Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:50:49.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:49 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[72459]: ts=2024-03-29T04:50:49.486Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:50:49.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:49 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[72459]: ts=2024-03-29T04:50:49.489Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-03-29T04:50:49.701 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:49 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[72459]: ts=2024-03-29T04:50:49.489Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-03-29T04:50:49.862 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:50.031 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:49 smithi181 bash[72405]: 79e961bfe034831b50ec306b60700047b537dd433482668a330da0dc476b3f4c 2024-03-29T04:50:50.032 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:49 smithi181 systemd[1]: Started Ceph alertmanager.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:50:50.032 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:49 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:50:49] ENGINE Bus STOPPING 2024-03-29T04:50:50.032 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:49 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:50:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:50:50.032 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:49 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:50:49] ENGINE Bus STOPPED 2024-03-29T04:50:50.032 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:49 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:50:49] ENGINE Bus STARTING 2024-03-29T04:50:50.032 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:50:50] ENGINE Serving on http://:::9283 2024-03-29T04:50:50.032 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:50:50] ENGINE Bus STARTED 2024-03-29T04:50:50.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[41183]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:50.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:50.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:50.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:50.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:50.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:50.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:50.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:50:50.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[41183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:50:50.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:50.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[41183]: Deploying daemon grafana.a on smithi187 2024-03-29T04:50:50.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[29765]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:50.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:50.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:50.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:50.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:50.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:50.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:50.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:50:50.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[29765]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:50:50.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:50.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:50 smithi181 ceph-mon[29765]: Deploying daemon grafana.a on smithi187 2024-03-29T04:50:51.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:50 smithi187 ceph-mon[33595]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:51.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:50 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:51.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:50 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:51.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:50 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:51.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:50 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:51.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:50 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:51.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:50 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:51.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:50 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:50:51.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:50 smithi187 ceph-mon[33595]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:50:51.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:50 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:51.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:50 smithi187 ceph-mon[33595]: Deploying daemon grafana.a on smithi187 2024-03-29T04:50:51.699 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[72459]: ts=2024-03-29T04:50:51.458Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000776343s 2024-03-29T04:50:51.741 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:50:51.741 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":64,"fsid":"dfcecab8-ed86-11ee-95d0-87774f69a715","created":"2024-03-29T04:44:49.014354+0000","modified":"2024-03-29T04:50:42.049307+0000","last_up_change":"2024-03-29T04:50:02.141186+0000","last_in_change":"2024-03-29T04:49:44.602034+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:48:01.551395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:50:37.400395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"26991441-a6cb-4694-aec3-0f793de6d551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6803","nonce":135202285}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6805","nonce":135202285}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6809","nonce":135202285}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6807","nonce":135202285}]},"public_addr":"172.21.15.181:6803/135202285","cluster_addr":"172.21.15.181:6805/135202285","heartbeat_back_addr":"172.21.15.181:6809/135202285","heartbeat_front_addr":"172.21.15.181:6807/135202285","state":["exists","up"]},{"osd":1,"uuid":"04b645ce-4eb8-43bc-9f28-e3be932f4b1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6811","nonce":3021678652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6813","nonce":3021678652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6817","nonce":3021678652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6815","nonce":3021678652}]},"public_addr":"172.21.15.181:6811/3021678652","cluster_addr":"172.21.15.181:6813/3021678652","heartbeat_back_addr":"172.21.15.181:6817/3021678652","heartbeat_front_addr":"172.21.15.181:6815/3021678652","state":["exists","up"]},{"osd":2,"uuid":"f72d641c-5d86-4d3a-bf90-d99c1c46a805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6819","nonce":352673207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6821","nonce":352673207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6825","nonce":352673207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6823","nonce":352673207}]},"public_addr":"172.21.15.181:6819/352673207","cluster_addr":"172.21.15.181:6821/352673207","heartbeat_back_addr":"172.21.15.181:6825/352673207","heartbeat_front_addr":"172.21.15.181:6823/352673207","state":["exists","up"]},{"osd":3,"uuid":"63ebbc1a-43b5-4990-87aa-f8f0ff975830","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6827","nonce":1785806254}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6829","nonce":1785806254}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6832","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6833","nonce":1785806254}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6831","nonce":1785806254}]},"public_addr":"172.21.15.181:6827/1785806254","cluster_addr":"172.21.15.181:6829/1785806254","heartbeat_back_addr":"172.21.15.181:6833/1785806254","heartbeat_front_addr":"172.21.15.181:6831/1785806254","state":["exists","up"]},{"osd":4,"uuid":"0a71f228-2cc8-4b83-b86d-62a2854a8533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6800","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6801","nonce":3700510703}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6802","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6803","nonce":3700510703}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6806","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6807","nonce":3700510703}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6804","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6805","nonce":3700510703}]},"public_addr":"172.21.15.187:6801/3700510703","cluster_addr":"172.21.15.187:6803/3700510703","heartbeat_back_addr":"172.21.15.187:6807/3700510703","heartbeat_front_addr":"172.21.15.187:6805/3700510703","state":["exists","up"]},{"osd":5,"uuid":"847657e4-7b9a-4eee-9df5-8e32d0d6543e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6808","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6809","nonce":989620923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6810","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6811","nonce":989620923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6814","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6815","nonce":989620923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6812","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6813","nonce":989620923}]},"public_addr":"172.21.15.187:6809/989620923","cluster_addr":"172.21.15.187:6811/989620923","heartbeat_back_addr":"172.21.15.187:6815/989620923","heartbeat_front_addr":"172.21.15.187:6813/989620923","state":["exists","up"]},{"osd":6,"uuid":"cead714e-ffc1-44a4-bef5-cfa772070c9f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6816","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6817","nonce":1708678714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6818","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6819","nonce":1708678714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6822","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6823","nonce":1708678714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6820","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6821","nonce":1708678714}]},"public_addr":"172.21.15.187:6817/1708678714","cluster_addr":"172.21.15.187:6819/1708678714","heartbeat_back_addr":"172.21.15.187:6823/1708678714","heartbeat_front_addr":"172.21.15.187:6821/1708678714","state":["exists","up"]},{"osd":7,"uuid":"9e2520cb-e4ba-4b2a-9206-8a250e7a373f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6824","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6825","nonce":2139604103}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6826","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6827","nonce":2139604103}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6830","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6831","nonce":2139604103}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6828","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6829","nonce":2139604103}]},"public_addr":"172.21.15.187:6825/2139604103","cluster_addr":"172.21.15.187:6827/2139604103","heartbeat_back_addr":"172.21.15.187:6831/2139604103","heartbeat_front_addr":"172.21.15.187:6829/2139604103","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:08.364426+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:33.120007+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:57.511900+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:48:22.235902+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:48:45.159568+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:49:10.475329+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:49:34.890974+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:50:00.243723+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.181:6800/3044637144":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/4244016576":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/3491810134":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/3153434425":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/114113941":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/2540916225":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/2863027222":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/19173868":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/1768629494":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6800/2863027222":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/2056930738":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/4052605721":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/1867140711":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6800/2540916225":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/3044637144":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/196688700":"2024-03-30T04:45:25.467994+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:50:52.066 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:51 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2009124322' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:50:52.066 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:51 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2009124322' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:50:52.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:51 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2009124322' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:50:52.459 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-29T04:50:52.459 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:50:52.711 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:53.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:52 smithi187 ceph-mon[33595]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:53.179 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:52 smithi181 ceph-mon[29765]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:53.179 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:52 smithi181 ceph-mon[41183]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:54.549 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:50:54.549 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":64,"fsid":"dfcecab8-ed86-11ee-95d0-87774f69a715","created":"2024-03-29T04:44:49.014354+0000","modified":"2024-03-29T04:50:42.049307+0000","last_up_change":"2024-03-29T04:50:02.141186+0000","last_in_change":"2024-03-29T04:49:44.602034+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:48:01.551395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:50:37.400395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"26991441-a6cb-4694-aec3-0f793de6d551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6803","nonce":135202285}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6805","nonce":135202285}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6809","nonce":135202285}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6807","nonce":135202285}]},"public_addr":"172.21.15.181:6803/135202285","cluster_addr":"172.21.15.181:6805/135202285","heartbeat_back_addr":"172.21.15.181:6809/135202285","heartbeat_front_addr":"172.21.15.181:6807/135202285","state":["exists","up"]},{"osd":1,"uuid":"04b645ce-4eb8-43bc-9f28-e3be932f4b1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6811","nonce":3021678652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6813","nonce":3021678652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6817","nonce":3021678652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6815","nonce":3021678652}]},"public_addr":"172.21.15.181:6811/3021678652","cluster_addr":"172.21.15.181:6813/3021678652","heartbeat_back_addr":"172.21.15.181:6817/3021678652","heartbeat_front_addr":"172.21.15.181:6815/3021678652","state":["exists","up"]},{"osd":2,"uuid":"f72d641c-5d86-4d3a-bf90-d99c1c46a805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6819","nonce":352673207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6821","nonce":352673207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6825","nonce":352673207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6823","nonce":352673207}]},"public_addr":"172.21.15.181:6819/352673207","cluster_addr":"172.21.15.181:6821/352673207","heartbeat_back_addr":"172.21.15.181:6825/352673207","heartbeat_front_addr":"172.21.15.181:6823/352673207","state":["exists","up"]},{"osd":3,"uuid":"63ebbc1a-43b5-4990-87aa-f8f0ff975830","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6827","nonce":1785806254}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6829","nonce":1785806254}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6832","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6833","nonce":1785806254}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6831","nonce":1785806254}]},"public_addr":"172.21.15.181:6827/1785806254","cluster_addr":"172.21.15.181:6829/1785806254","heartbeat_back_addr":"172.21.15.181:6833/1785806254","heartbeat_front_addr":"172.21.15.181:6831/1785806254","state":["exists","up"]},{"osd":4,"uuid":"0a71f228-2cc8-4b83-b86d-62a2854a8533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6800","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6801","nonce":3700510703}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6802","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6803","nonce":3700510703}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6806","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6807","nonce":3700510703}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6804","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6805","nonce":3700510703}]},"public_addr":"172.21.15.187:6801/3700510703","cluster_addr":"172.21.15.187:6803/3700510703","heartbeat_back_addr":"172.21.15.187:6807/3700510703","heartbeat_front_addr":"172.21.15.187:6805/3700510703","state":["exists","up"]},{"osd":5,"uuid":"847657e4-7b9a-4eee-9df5-8e32d0d6543e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6808","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6809","nonce":989620923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6810","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6811","nonce":989620923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6814","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6815","nonce":989620923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6812","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6813","nonce":989620923}]},"public_addr":"172.21.15.187:6809/989620923","cluster_addr":"172.21.15.187:6811/989620923","heartbeat_back_addr":"172.21.15.187:6815/989620923","heartbeat_front_addr":"172.21.15.187:6813/989620923","state":["exists","up"]},{"osd":6,"uuid":"cead714e-ffc1-44a4-bef5-cfa772070c9f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6816","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6817","nonce":1708678714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6818","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6819","nonce":1708678714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6822","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6823","nonce":1708678714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6820","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6821","nonce":1708678714}]},"public_addr":"172.21.15.187:6817/1708678714","cluster_addr":"172.21.15.187:6819/1708678714","heartbeat_back_addr":"172.21.15.187:6823/1708678714","heartbeat_front_addr":"172.21.15.187:6821/1708678714","state":["exists","up"]},{"osd":7,"uuid":"9e2520cb-e4ba-4b2a-9206-8a250e7a373f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6824","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6825","nonce":2139604103}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6826","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6827","nonce":2139604103}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6830","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6831","nonce":2139604103}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6828","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6829","nonce":2139604103}]},"public_addr":"172.21.15.187:6825/2139604103","cluster_addr":"172.21.15.187:6827/2139604103","heartbeat_back_addr":"172.21.15.187:6831/2139604103","heartbeat_front_addr":"172.21.15.187:6829/2139604103","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:08.364426+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:33.120007+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:57.511900+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:48:22.235902+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:48:45.159568+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:49:10.475329+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:49:34.890974+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:50:00.243723+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.181:6800/3044637144":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/4244016576":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/3491810134":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/3153434425":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/114113941":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/2540916225":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/2863027222":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/19173868":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/1768629494":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6800/2863027222":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/2056930738":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/4052605721":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/1867140711":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6800/2540916225":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/3044637144":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/196688700":"2024-03-30T04:45:25.467994+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:50:54.906 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:54 smithi181 ceph-mon[29765]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:54.906 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:54 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:54.906 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:54 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/675109938' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:50:54.907 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:54 smithi181 ceph-mon[41183]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:54.907 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:54 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:54.907 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:54 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/675109938' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:50:55.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:54 smithi187 ceph-mon[33595]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:55.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:54 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:50:55.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:54 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/675109938' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:50:55.597 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-29T04:50:55.597 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-29T04:50:55.597 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-29T04:50:55.598 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-29T04:50:55.598 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-29T04:50:55.599 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-29T04:50:55.599 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-29T04:50:55.599 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-29T04:50:56.049 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:56.165 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:56.169 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:56.169 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:56.170 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:56.171 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:56.171 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:56.175 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:50:56.806 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:56 smithi181 ceph-mon[29765]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:56.806 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:56 smithi181 ceph-mon[41183]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:57.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:56 smithi187 ceph-mon[33595]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:58.794 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:50:58 smithi181 ceph-mon[41183]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:58.794 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:50:58 smithi181 ceph-mon[29765]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:58.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:50:58 smithi187 ceph-mon[33595]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:58.879 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:50:58 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:50:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:50:59.785 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:50:59 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[72459]: ts=2024-03-29T04:50:59.460Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003003442s 2024-03-29T04:51:00.697 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:00 smithi181 ceph-mon[41183]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:00.697 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:00 smithi181 ceph-mon[29765]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:00.935 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:00 smithi187 ceph-mon[33595]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:02.782 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:02 smithi187 ceph-mon[33595]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:02.783 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:02 smithi187 podman[65704]: 2024-03-29 04:51:02.647884018 +0000 UTC m=+0.150111918 container create 8bb6c690e335fe0559dedc92ba8a3ee9a2e7d0064fd9beae8fe3dc0ce2dae66b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , distribution-scope=public, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, architecture=x86_64, com.redhat.component=ubi8-container, io.openshift.expose-services=, version=8.5, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, vcs-type=git) 2024-03-29T04:51:02.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:02 smithi181 ceph-mon[41183]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:02.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:02 smithi181 ceph-mon[29765]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:03.132 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:02 smithi187 podman[65704]: 2024-03-29 04:51:02.877516628 +0000 UTC m=+0.379744528 container init 8bb6c690e335fe0559dedc92ba8a3ee9a2e7d0064fd9beae8fe3dc0ce2dae66b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, maintainer=Paul Cuzner , vcs-type=git, description=Ceph Grafana Container, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.component=ubi8-container, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, io.openshift.expose-services=, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public) 2024-03-29T04:51:03.132 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:02 smithi187 podman[65704]: 2024-03-29 04:51:02.882576459 +0000 UTC m=+0.384804360 container start 8bb6c690e335fe0559dedc92ba8a3ee9a2e7d0064fd9beae8fe3dc0ce2dae66b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, io.openshift.expose-services=, version=8.5, vcs-type=git, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.component=ubi8-container, name=ubi8, distribution-scope=public, description=Ceph Grafana Container, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-03-29T04:51:03.133 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:02 smithi187 bash[65704]: 8bb6c690e335fe0559dedc92ba8a3ee9a2e7d0064fd9beae8fe3dc0ce2dae66b 2024-03-29T04:51:03.133 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:02 smithi187 systemd[1]: Started Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:51:03.133 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=settings t=2024-03-29T04:51:03.038019728Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-03-29T04:51:03.133 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=settings t=2024-03-29T04:51:03.03820257Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-03-29T04:51:03.133 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=settings t=2024-03-29T04:51:03.038221645Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-03-29T04:51:03.133 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=settings t=2024-03-29T04:51:03.0382289Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-03-29T04:51:03.134 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=settings t=2024-03-29T04:51:03.038235818Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-03-29T04:51:03.134 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=settings t=2024-03-29T04:51:03.038243348Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-03-29T04:51:03.134 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=settings t=2024-03-29T04:51:03.038249165Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-03-29T04:51:03.134 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=settings t=2024-03-29T04:51:03.038255183Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-03-29T04:51:03.134 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=settings t=2024-03-29T04:51:03.038261727Z level=info msg="Path Home" path=/usr/share/grafana 2024-03-29T04:51:03.134 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=settings t=2024-03-29T04:51:03.038267678Z level=info msg="Path Data" path=/var/lib/grafana 2024-03-29T04:51:03.134 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=settings t=2024-03-29T04:51:03.038273338Z level=info msg="Path Logs" path=/var/log/grafana 2024-03-29T04:51:03.134 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=settings t=2024-03-29T04:51:03.038278859Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-03-29T04:51:03.134 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=settings t=2024-03-29T04:51:03.038284645Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-03-29T04:51:03.134 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=settings t=2024-03-29T04:51:03.038290566Z level=info msg="App mode production" 2024-03-29T04:51:03.135 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=sqlstore t=2024-03-29T04:51:03.038336504Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-03-29T04:51:03.135 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=sqlstore t=2024-03-29T04:51:03.038352538Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-03-29T04:51:03.135 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.039191762Z level=info msg="Starting DB migrations" 2024-03-29T04:51:03.135 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.039264Z level=info msg="Executing migration" id="create migration_log table" 2024-03-29T04:51:03.135 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.03979941Z level=info msg="Executing migration" id="create user table" 2024-03-29T04:51:03.135 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.040241306Z level=info msg="Executing migration" id="add unique index user.login" 2024-03-29T04:51:03.135 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.041254564Z level=info msg="Executing migration" id="add unique index user.email" 2024-03-29T04:51:03.135 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.041958032Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-03-29T04:51:03.135 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.042926332Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-03-29T04:51:03.135 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.043627657Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-03-29T04:51:03.135 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.044487418Z level=info msg="Executing migration" id="create user table v2" 2024-03-29T04:51:03.135 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.045402861Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-03-29T04:51:03.135 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.046255205Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-03-29T04:51:03.135 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.046753276Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-03-29T04:51:03.135 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.047534189Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-03-29T04:51:03.136 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.048384102Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-03-29T04:51:03.136 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.049289111Z level=info msg="Executing migration" id="Update user table charset" 2024-03-29T04:51:03.136 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.050303724Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-03-29T04:51:03.136 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.050987626Z level=info msg="Executing migration" id="Add missing user data" 2024-03-29T04:51:03.136 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.051341435Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-03-29T04:51:03.136 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.052198467Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-03-29T04:51:03.136 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.053068344Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-03-29T04:51:03.136 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.055803291Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-03-29T04:51:03.136 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.056961296Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-03-29T04:51:03.136 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.059720972Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-03-29T04:51:03.136 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.062652303Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-03-29T04:51:03.136 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.065321375Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-03-29T04:51:03.136 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.068057743Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-03-29T04:51:03.136 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.07096362Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-03-29T04:51:03.136 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.073686272Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-03-29T04:51:03.137 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.07634511Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-03-29T04:51:03.137 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.079766621Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-03-29T04:51:03.137 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.082519584Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-03-29T04:51:03.137 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.085336171Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-03-29T04:51:03.137 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.088139413Z level=info msg="Executing migration" id="create temp_user v2" 2024-03-29T04:51:03.137 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.091055691Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-03-29T04:51:03.137 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.093670027Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-03-29T04:51:03.137 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.096572064Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-03-29T04:51:03.137 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.099428654Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-03-29T04:51:03.137 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.102125982Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-03-29T04:51:03.137 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.104631592Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-03-29T04:51:03.137 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.107316681Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-03-29T04:51:03.137 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.10990868Z level=info msg="Executing migration" id="create star table" 2024-03-29T04:51:03.137 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.110284167Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-03-29T04:51:03.137 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.113047182Z level=info msg="Executing migration" id="create org table v1" 2024-03-29T04:51:03.138 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.11556347Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-03-29T04:51:03.138 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.118238509Z level=info msg="Executing migration" id="create org_user table v1" 2024-03-29T04:51:03.138 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.120974035Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-03-29T04:51:03.138 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.123648558Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-03-29T04:51:03.138 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.126466532Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-03-29T04:51:03.138 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.126998202Z level=info msg="Executing migration" id="Update org table charset" 2024-03-29T04:51:03.138 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.129866767Z level=info msg="Executing migration" id="Update org_user table charset" 2024-03-29T04:51:03.385 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.132476194Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-03-29T04:51:03.385 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.135034222Z level=info msg="Executing migration" id="create dashboard table" 2024-03-29T04:51:03.385 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.13541809Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-03-29T04:51:03.385 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.138167918Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-03-29T04:51:03.386 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.140810804Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-03-29T04:51:03.386 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.141604881Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-03-29T04:51:03.386 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.142015816Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-03-29T04:51:03.386 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.142915251Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-03-29T04:51:03.386 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.143812426Z level=info msg="Executing migration" id="create dashboard v2" 2024-03-29T04:51:03.386 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.144729773Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-03-29T04:51:03.386 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.145341986Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-03-29T04:51:03.386 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.146160174Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-03-29T04:51:03.386 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.147116487Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-03-29T04:51:03.386 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.147841761Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-03-29T04:51:03.386 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.148148717Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-03-29T04:51:03.386 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.14906399Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-03-29T04:51:03.386 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.149760887Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-03-29T04:51:03.387 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.15034776Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-03-29T04:51:03.387 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.151152019Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-03-29T04:51:03.387 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.151639931Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-03-29T04:51:03.387 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.15246793Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-03-29T04:51:03.387 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.153305763Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-03-29T04:51:03.387 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.154125226Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-03-29T04:51:03.387 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.154408287Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-03-29T04:51:03.387 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.155316362Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-03-29T04:51:03.387 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.156128365Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-03-29T04:51:03.387 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.156986169Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-03-29T04:51:03.387 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.157503672Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-03-29T04:51:03.387 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.15828938Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-03-29T04:51:03.387 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.158730054Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-03-29T04:51:03.387 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.159192908Z level=info msg="Executing migration" id="Update dashboard title length" 2024-03-29T04:51:03.388 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.160144613Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-03-29T04:51:03.388 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.160662698Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-03-29T04:51:03.388 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.161524488Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-03-29T04:51:03.388 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.162607884Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-03-29T04:51:03.388 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.163453193Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-03-29T04:51:03.388 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.163868935Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-03-29T04:51:03.388 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.164800282Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-03-29T04:51:03.390 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.165486975Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-03-29T04:51:03.390 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.166241472Z level=info msg="Executing migration" id="Add check_sum column" 2024-03-29T04:51:03.390 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.166784675Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-03-29T04:51:03.390 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.167200873Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-03-29T04:51:03.390 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.168013092Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-03-29T04:51:03.390 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.168827154Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-03-29T04:51:03.390 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.169831768Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-03-29T04:51:03.390 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.170443027Z level=info msg="Executing migration" id="create data_source table" 2024-03-29T04:51:03.390 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.171213724Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-03-29T04:51:03.390 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.171981666Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-03-29T04:51:03.390 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.172385107Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-03-29T04:51:03.391 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.172736153Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-03-29T04:51:03.391 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.173128717Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-03-29T04:51:03.391 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.17409125Z level=info msg="Executing migration" id="create data_source table v2" 2024-03-29T04:51:03.391 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.175048757Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-03-29T04:51:03.391 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.175808163Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-03-29T04:51:03.391 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.176284867Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-03-29T04:51:03.391 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.177054157Z level=info msg="Executing migration" id="Add column with_credentials" 2024-03-29T04:51:03.391 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.177984026Z level=info msg="Executing migration" id="Add secure json data column" 2024-03-29T04:51:03.391 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.17871127Z level=info msg="Executing migration" id="Update data_source table charset" 2024-03-29T04:51:03.391 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.179008804Z level=info msg="Executing migration" id="Update initial version to 1" 2024-03-29T04:51:03.391 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.179330313Z level=info msg="Executing migration" id="Add read_only data column" 2024-03-29T04:51:03.391 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.179830514Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-03-29T04:51:03.391 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.180164772Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-03-29T04:51:03.391 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.18066576Z level=info msg="Executing migration" id="Add uid column" 2024-03-29T04:51:03.391 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.181547392Z level=info msg="Executing migration" id="Update uid value" 2024-03-29T04:51:03.392 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.181879716Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-03-29T04:51:03.392 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.182695782Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-03-29T04:51:03.392 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.1854456Z level=info msg="Executing migration" id="create api_key table" 2024-03-29T04:51:03.392 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.188189747Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-03-29T04:51:03.392 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.188613046Z level=info msg="Executing migration" id="add index api_key.key" 2024-03-29T04:51:03.392 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.191316718Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-03-29T04:51:03.392 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.194043983Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-03-29T04:51:03.392 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.196822958Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-03-29T04:51:03.392 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.199485159Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-03-29T04:51:03.392 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.202165615Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-03-29T04:51:03.392 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.204972376Z level=info msg="Executing migration" id="create api_key table v2" 2024-03-29T04:51:03.392 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.207722502Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-03-29T04:51:03.392 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.210495981Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-03-29T04:51:03.392 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.212972367Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-03-29T04:51:03.393 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.215818118Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-03-29T04:51:03.393 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.216278502Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-03-29T04:51:03.393 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.218430488Z level=info msg="Executing migration" id="Update api_key table charset" 2024-03-29T04:51:03.393 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.218894232Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-03-29T04:51:03.393 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.22129529Z level=info msg="Executing migration" id="Add service account foreign key" 2024-03-29T04:51:03.393 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.224103057Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-03-29T04:51:03.393 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.226988825Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-03-29T04:51:03.393 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.229512592Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-03-29T04:51:03.393 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.232020536Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-03-29T04:51:03.393 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.234732752Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-03-29T04:51:03.393 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.237544722Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-03-29T04:51:03.393 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.2401243Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-03-29T04:51:03.393 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.243129238Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-03-29T04:51:03.394 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.245456906Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-03-29T04:51:03.394 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.247940212Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-03-29T04:51:03.394 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.250658049Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-03-29T04:51:03.394 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.253365015Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-03-29T04:51:03.394 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.255958285Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-03-29T04:51:03.394 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.25864791Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-03-29T04:51:03.394 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.261346044Z level=info msg="Executing migration" id="create quota table v1" 2024-03-29T04:51:03.394 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.26414124Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-03-29T04:51:03.394 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.266813358Z level=info msg="Executing migration" id="Update quota table charset" 2024-03-29T04:51:03.394 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.267139017Z level=info msg="Executing migration" id="create plugin_setting table" 2024-03-29T04:51:03.394 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.269534192Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-03-29T04:51:03.394 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.272139969Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-03-29T04:51:03.394 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.272910837Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-03-29T04:51:03.394 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.273223669Z level=info msg="Executing migration" id="create session table" 2024-03-29T04:51:03.395 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.273717616Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-03-29T04:51:03.395 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.274503684Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-03-29T04:51:03.395 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.274877377Z level=info msg="Executing migration" id="create playlist table v2" 2024-03-29T04:51:03.395 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.275276582Z level=info msg="Executing migration" id="create playlist item table v2" 2024-03-29T04:51:03.395 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.276055579Z level=info msg="Executing migration" id="Update playlist table charset" 2024-03-29T04:51:03.395 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.276953367Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-03-29T04:51:03.395 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.277252913Z level=info msg="Executing migration" id="drop preferences table v2" 2024-03-29T04:51:03.395 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.277520837Z level=info msg="Executing migration" id="drop preferences table v3" 2024-03-29T04:51:03.395 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.277794156Z level=info msg="Executing migration" id="create preferences table v3" 2024-03-29T04:51:03.395 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.278206206Z level=info msg="Executing migration" id="Update preferences table charset" 2024-03-29T04:51:03.395 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.279020917Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-03-29T04:51:03.395 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.279611133Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-03-29T04:51:03.396 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.27991872Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-03-29T04:51:03.396 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.28050701Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-03-29T04:51:03.396 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.281349731Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-03-29T04:51:03.396 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.282187297Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-03-29T04:51:03.396 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.282692971Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-03-29T04:51:03.396 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.283585706Z level=info msg="Executing migration" id="create alert table v1" 2024-03-29T04:51:03.396 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.284469433Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-03-29T04:51:03.396 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.285107223Z level=info msg="Executing migration" id="add index alert state" 2024-03-29T04:51:03.396 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.285923257Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-03-29T04:51:03.396 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.286375862Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-03-29T04:51:03.396 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.28674955Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-03-29T04:51:03.396 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.287183431Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-03-29T04:51:03.396 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.287594383Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-03-29T04:51:03.396 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.289066046Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-03-29T04:51:03.397 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.289445576Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-03-29T04:51:03.397 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.289903417Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-03-29T04:51:03.397 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.290226419Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-03-29T04:51:03.397 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.290563253Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-03-29T04:51:03.397 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.290930764Z level=info msg="Executing migration" id="Add column is_default" 2024-03-29T04:51:03.397 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.291588536Z level=info msg="Executing migration" id="Add column frequency" 2024-03-29T04:51:03.397 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.292243743Z level=info msg="Executing migration" id="Add column send_reminder" 2024-03-29T04:51:03.397 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.292902491Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-03-29T04:51:03.397 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.293562741Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-03-29T04:51:03.397 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.293985845Z level=info msg="Executing migration" id="Update alert table charset" 2024-03-29T04:51:03.397 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.294283967Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-03-29T04:51:03.397 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.294559327Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-03-29T04:51:03.397 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.294980125Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-03-29T04:51:03.397 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.295427758Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-03-29T04:51:03.398 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.295806278Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-03-29T04:51:03.398 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.296206538Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-03-29T04:51:03.398 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.296643427Z level=info msg="Executing migration" id="Add for to alert table" 2024-03-29T04:51:03.398 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.297379526Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-03-29T04:51:03.398 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.298016894Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-03-29T04:51:03.398 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.298349179Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-03-29T04:51:03.398 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.298784304Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-03-29T04:51:03.398 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.299199213Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-03-29T04:51:03.398 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.299856427Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-03-29T04:51:03.398 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.300183873Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-03-29T04:51:03.398 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.300587935Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-03-29T04:51:03.398 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.301006975Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-03-29T04:51:03.398 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.301343896Z level=info msg="Executing migration" id="create annotation table v5" 2024-03-29T04:51:03.398 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.301771223Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-03-29T04:51:03.399 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.302552672Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-03-29T04:51:03.399 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.302977822Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-03-29T04:51:03.399 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.303418347Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-03-29T04:51:03.399 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.303875996Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-03-29T04:51:03.399 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.304738457Z level=info msg="Executing migration" id="Update annotation table charset" 2024-03-29T04:51:03.399 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.305069784Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-03-29T04:51:03.399 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.30573828Z level=info msg="Executing migration" id="Drop category_id index" 2024-03-29T04:51:03.399 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.30665967Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-03-29T04:51:03.399 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.307527356Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-03-29T04:51:03.399 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.308366983Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-03-29T04:51:03.399 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.309390793Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-03-29T04:51:03.399 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.310059895Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-03-29T04:51:03.399 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.311704841Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-03-29T04:51:03.399 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.312688919Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-03-29T04:51:03.399 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.313535782Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-03-29T04:51:03.400 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.314326671Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-03-29T04:51:03.400 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.315260254Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-03-29T04:51:03.400 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.315966994Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-03-29T04:51:03.400 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.316641156Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-03-29T04:51:03.400 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.317512036Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-03-29T04:51:03.400 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.318510191Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-03-29T04:51:03.400 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.321172871Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-03-29T04:51:03.400 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.323998207Z level=info msg="Executing migration" id="Add epoch_end column" 2024-03-29T04:51:03.400 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.326751773Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-03-29T04:51:03.400 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.329569678Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-03-29T04:51:03.400 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.332320511Z level=info msg="Executing migration" id="Move region to single row" 2024-03-29T04:51:03.400 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.335114645Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-03-29T04:51:03.400 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.337969304Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-03-29T04:51:03.400 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.340922267Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-03-29T04:51:03.401 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.34373909Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-03-29T04:51:03.401 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.346638275Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-03-29T04:51:03.401 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.349347148Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-03-29T04:51:03.401 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.352116756Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-03-29T04:51:03.401 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.354893805Z level=info msg="Executing migration" id="create test_data table" 2024-03-29T04:51:03.401 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.35769257Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-03-29T04:51:03.401 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.360390222Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-03-29T04:51:03.401 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.363254554Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-03-29T04:51:03.401 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.365894519Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-03-29T04:51:03.401 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.368695911Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-03-29T04:51:03.401 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.371415606Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-03-29T04:51:03.401 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.374095719Z level=info msg="Executing migration" id="create team table" 2024-03-29T04:51:03.401 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.376696785Z level=info msg="Executing migration" id="add index team.org_id" 2024-03-29T04:51:03.401 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.379380207Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-03-29T04:51:03.402 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.3820558Z level=info msg="Executing migration" id="create team member table" 2024-03-29T04:51:03.402 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.384601056Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-03-29T04:51:03.637 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.38733631Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-03-29T04:51:03.637 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.390104024Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-03-29T04:51:03.637 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.392796394Z level=info msg="Executing migration" id="Add column email to team table" 2024-03-29T04:51:03.638 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.395622925Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-03-29T04:51:03.638 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.398305478Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-03-29T04:51:03.638 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.401065038Z level=info msg="Executing migration" id="create dashboard acl table" 2024-03-29T04:51:03.638 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.403688567Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-03-29T04:51:03.638 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.406279468Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-03-29T04:51:03.638 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.408930502Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-03-29T04:51:03.638 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.411486465Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-03-29T04:51:03.638 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.414167841Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-03-29T04:51:03.638 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.414871373Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-03-29T04:51:03.638 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.415325731Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-03-29T04:51:03.638 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.416162674Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-03-29T04:51:03.638 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.417014231Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-03-29T04:51:03.638 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.41779823Z level=info msg="Executing migration" id="create tag table" 2024-03-29T04:51:03.639 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.418205161Z level=info msg="Executing migration" id="add index tag.key_value" 2024-03-29T04:51:03.639 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.419037617Z level=info msg="Executing migration" id="create login attempt table" 2024-03-29T04:51:03.639 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.420033163Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-03-29T04:51:03.639 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.420736545Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-03-29T04:51:03.639 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.421204794Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-03-29T04:51:03.639 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.423201464Z level=info msg="Executing migration" id="create login_attempt v2" 2024-03-29T04:51:03.639 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.424025399Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-03-29T04:51:03.639 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.425077976Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-03-29T04:51:03.639 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.425478473Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-03-29T04:51:03.639 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.426311751Z level=info msg="Executing migration" id="create user auth table" 2024-03-29T04:51:03.639 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.426729134Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-03-29T04:51:03.639 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.427213095Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-03-29T04:51:03.639 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.42802296Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-03-29T04:51:03.639 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.428804463Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-03-29T04:51:03.639 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.429683055Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-03-29T04:51:03.640 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.430552972Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-03-29T04:51:03.640 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.431398877Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-03-29T04:51:03.640 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.431835636Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-03-29T04:51:03.640 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.43267164Z level=info msg="Executing migration" id="create server_lock table" 2024-03-29T04:51:03.640 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.433545069Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-03-29T04:51:03.640 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.434390459Z level=info msg="Executing migration" id="create user auth token table" 2024-03-29T04:51:03.640 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.434830225Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-03-29T04:51:03.641 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.435672526Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-03-29T04:51:03.641 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.436519219Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-03-29T04:51:03.641 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.437550496Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-03-29T04:51:03.641 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.438379608Z level=info msg="Executing migration" id="create cache_data table" 2024-03-29T04:51:03.641 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.439360847Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-03-29T04:51:03.641 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.440094281Z level=info msg="Executing migration" id="create short_url table v1" 2024-03-29T04:51:03.641 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.440955968Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-03-29T04:51:03.641 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.441380271Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-03-29T04:51:03.642 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.444117919Z level=info msg="Executing migration" id="delete alert_definition table" 2024-03-29T04:51:03.642 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.444457317Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-03-29T04:51:03.642 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.447037201Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-03-29T04:51:03.642 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.449772294Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-03-29T04:51:03.642 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.450237284Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-03-29T04:51:03.642 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.453076794Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-03-29T04:51:03.642 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.455844783Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-03-29T04:51:03.642 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.458680431Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-03-29T04:51:03.642 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.459217207Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-03-29T04:51:03.642 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.462259781Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-03-29T04:51:03.642 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.464944173Z level=info msg="Executing migration" id="drop alert_definition table" 2024-03-29T04:51:03.642 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.46766355Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-03-29T04:51:03.642 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.468003279Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-03-29T04:51:03.643 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.470736659Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-03-29T04:51:03.643 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.473591333Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-03-29T04:51:03.643 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.476353499Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-03-29T04:51:03.643 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.476685903Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-03-29T04:51:03.643 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.479483123Z level=info msg="Executing migration" id="create alert_instance table" 2024-03-29T04:51:03.643 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.482158679Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-03-29T04:51:03.643 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.484764532Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-03-29T04:51:03.643 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.485224918Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-03-29T04:51:03.643 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.487966512Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-03-29T04:51:03.643 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.49062936Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-03-29T04:51:03.643 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.493261645Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-03-29T04:51:03.643 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.496618264Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-03-29T04:51:03.643 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.499949187Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-03-29T04:51:03.644 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.502593675Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-03-29T04:51:03.644 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.505326039Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-03-29T04:51:03.644 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.508019744Z level=info msg="Executing migration" id="create alert_rule table" 2024-03-29T04:51:03.644 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.510934406Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-03-29T04:51:03.644 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.513572081Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-03-29T04:51:03.644 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.51616257Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-03-29T04:51:03.644 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.519044962Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-03-29T04:51:03.644 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.521715282Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-03-29T04:51:03.644 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.524326856Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-03-29T04:51:03.644 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.525172033Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-03-29T04:51:03.644 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.52601715Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-03-29T04:51:03.644 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.527014245Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-03-29T04:51:03.644 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.527726181Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-03-29T04:51:03.644 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.52858103Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-03-29T04:51:03.644 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.529455018Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-03-29T04:51:03.645 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.530345702Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-03-29T04:51:03.645 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.531194883Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-03-29T04:51:03.645 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.532005982Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-03-29T04:51:03.645 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.532918679Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-03-29T04:51:03.645 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.533741268Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-03-29T04:51:03.645 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.534707148Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-03-29T04:51:03.645 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.535510178Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-03-29T04:51:03.645 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.536227485Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-03-29T04:51:03.645 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.537107775Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-03-29T04:51:03.645 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.537993929Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-03-29T04:51:03.645 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.538854297Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-03-29T04:51:03.645 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.539710532Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-03-29T04:51:03.645 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.540584063Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-03-29T04:51:03.645 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.541595351Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-03-29T04:51:03.646 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.542479814Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-03-29T04:51:03.646 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.544759056Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-03-29T04:51:03.646 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.547425189Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-03-29T04:51:03.646 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.550270886Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-03-29T04:51:03.646 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.552859407Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-03-29T04:51:03.646 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.555679597Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-03-29T04:51:03.646 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.55841162Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-03-29T04:51:03.646 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.56125661Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-03-29T04:51:03.646 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.563986269Z level=info msg="Executing migration" id="create provenance_type table" 2024-03-29T04:51:03.646 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.566795393Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-03-29T04:51:03.646 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.569492786Z level=info msg="Executing migration" id="create alert_image table" 2024-03-29T04:51:03.646 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.572160625Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-03-29T04:51:03.646 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.575060939Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-03-29T04:51:03.647 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.577945788Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-03-29T04:51:03.647 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.580604668Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-03-29T04:51:03.647 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.581039731Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-03-29T04:51:03.647 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.581114517Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-03-29T04:51:03.647 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.583841493Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-03-29T04:51:03.647 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.586564917Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-03-29T04:51:03.647 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.589002486Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-03-29T04:51:03.647 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.589118382Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-03-29T04:51:03.647 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.589230484Z level=info msg="alerts found to migrate" alerts=0 2024-03-29T04:51:03.647 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.591836315Z level=info msg="Executing migration" id="create library_element table v1" 2024-03-29T04:51:03.647 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.594625262Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-03-29T04:51:03.647 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.597116266Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-03-29T04:51:03.647 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.599863001Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-03-29T04:51:03.647 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.602360845Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-03-29T04:51:03.647 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.605042845Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-03-29T04:51:03.648 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.607770694Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-03-29T04:51:03.648 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.610380042Z level=info msg="Executing migration" id="create data_keys table" 2024-03-29T04:51:03.648 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.613003742Z level=info msg="Executing migration" id="create secrets table" 2024-03-29T04:51:03.648 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.615849031Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-03-29T04:51:03.648 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.620203712Z level=info msg="Executing migration" id="add name column into data_keys" 2024-03-29T04:51:03.648 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.623041863Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-03-29T04:51:03.648 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.625839984Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-03-29T04:51:03.648 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.630108216Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-03-29T04:51:03.648 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.634271136Z level=info msg="Executing migration" id="create kv_store table v1" 2024-03-29T04:51:03.648 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.63506596Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-03-29T04:51:03.648 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.635917354Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-03-29T04:51:03.648 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.636693135Z level=info msg="Executing migration" id="create permission table" 2024-03-29T04:51:04.049 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.637566982Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-03-29T04:51:04.049 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.63836206Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-03-29T04:51:04.049 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.63921666Z level=info msg="Executing migration" id="create role table" 2024-03-29T04:51:04.050 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.640000177Z level=info msg="Executing migration" id="add column display_name" 2024-03-29T04:51:04.050 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.640940678Z level=info msg="Executing migration" id="add column group_name" 2024-03-29T04:51:04.050 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.6418685Z level=info msg="Executing migration" id="add index role.org_id" 2024-03-29T04:51:04.050 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.642693733Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-03-29T04:51:04.050 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.643480612Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-03-29T04:51:04.050 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.644272463Z level=info msg="Executing migration" id="create team role table" 2024-03-29T04:51:04.050 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.645086562Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-03-29T04:51:04.050 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.646023198Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-03-29T04:51:04.050 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.646766215Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-03-29T04:51:04.050 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.647584569Z level=info msg="Executing migration" id="create user role table" 2024-03-29T04:51:04.050 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.648549364Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-03-29T04:51:04.050 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.649266185Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-03-29T04:51:04.050 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.650187335Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-03-29T04:51:04.050 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.65091732Z level=info msg="Executing migration" id="create builtin role table" 2024-03-29T04:51:04.051 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.651933196Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-03-29T04:51:04.051 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.652681196Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-03-29T04:51:04.051 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.653606774Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-03-29T04:51:04.051 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.654659105Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-03-29T04:51:04.051 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.65515407Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-03-29T04:51:04.051 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.655895732Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-03-29T04:51:04.051 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.656340851Z level=info msg="Executing migration" id="add unique index role.uid" 2024-03-29T04:51:04.051 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.656747201Z level=info msg="Executing migration" id="create seed assignment table" 2024-03-29T04:51:04.051 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.657170768Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-03-29T04:51:04.051 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.657605139Z level=info msg="Executing migration" id="add column hidden to role table" 2024-03-29T04:51:04.051 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.658588841Z level=info msg="Executing migration" id="create query_history table v1" 2024-03-29T04:51:04.051 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.65899676Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-03-29T04:51:04.051 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.659442709Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-03-29T04:51:04.052 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.659719015Z level=info msg="Executing migration" id="teams permissions migration" 2024-03-29T04:51:04.052 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.660144351Z level=info msg="Executing migration" id="dashboard permissions" 2024-03-29T04:51:04.052 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.660605004Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-03-29T04:51:04.052 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.660959727Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-03-29T04:51:04.052 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.661286576Z level=info msg="Executing migration" id="alerting notification permissions" 2024-03-29T04:51:04.052 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.661620115Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-03-29T04:51:04.052 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.66201522Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-03-29T04:51:04.052 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.6624508Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-03-29T04:51:04.053 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.66343279Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-03-29T04:51:04.053 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.663735835Z level=info msg="Executing migration" id="create correlation table v1" 2024-03-29T04:51:04.053 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.664174914Z level=info msg="Executing migration" id="add index correlations.uid" 2024-03-29T04:51:04.053 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.664607601Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-03-29T04:51:04.053 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.665040664Z level=info msg="Executing migration" id="add correlation config column" 2024-03-29T04:51:04.053 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.666018919Z level=info msg="Executing migration" id="create entity_events table" 2024-03-29T04:51:04.053 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.666446813Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-03-29T04:51:04.053 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.666873138Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-03-29T04:51:04.053 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.666974804Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-03-29T04:51:04.054 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.667265294Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-03-29T04:51:04.054 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.667351062Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-03-29T04:51:04.054 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.667608059Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-03-29T04:51:04.054 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.667968183Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-03-29T04:51:04.054 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.668381033Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-03-29T04:51:04.054 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.668805881Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-03-29T04:51:04.054 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.669246806Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-03-29T04:51:04.054 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.669661681Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-03-29T04:51:04.054 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.670094654Z level=info msg="Executing migration" id="Drop public config table" 2024-03-29T04:51:04.054 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.670495296Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-03-29T04:51:04.054 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.670894349Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-03-29T04:51:04.055 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.671327743Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-03-29T04:51:04.055 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.671764344Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-03-29T04:51:04.055 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.67224178Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-03-29T04:51:04.055 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.67520739Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-03-29T04:51:04.055 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.676218226Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-03-29T04:51:04.055 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.677226245Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-03-29T04:51:04.055 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.677571175Z level=info msg="Executing migration" id="add share column" 2024-03-29T04:51:04.055 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.678605029Z level=info msg="Executing migration" id="create default alerting folders" 2024-03-29T04:51:04.055 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.679096011Z level=info msg="Executing migration" id="create file table" 2024-03-29T04:51:04.055 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.679545221Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-03-29T04:51:04.055 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.680335749Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-03-29T04:51:04.055 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.680852401Z level=info msg="Executing migration" id="create file_meta table" 2024-03-29T04:51:04.055 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.681958552Z level=info msg="Executing migration" id="file table idx: path key" 2024-03-29T04:51:04.056 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.682554361Z level=info msg="Executing migration" id="set path collation in file table" 2024-03-29T04:51:04.056 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.683350882Z level=info msg="Executing migration" id="managed permissions migration" 2024-03-29T04:51:04.056 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.683775403Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-03-29T04:51:04.056 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.684219915Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-03-29T04:51:04.056 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.685087113Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-03-29T04:51:04.056 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.687801064Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-03-29T04:51:04.056 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.690685093Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-03-29T04:51:04.056 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.693387972Z level=info msg="Executing migration" id="update group index for alert rules" 2024-03-29T04:51:04.056 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.696049835Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-03-29T04:51:04.056 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.69855513Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-03-29T04:51:04.056 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.70144635Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-03-29T04:51:04.056 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.704073594Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-03-29T04:51:04.056 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.70663294Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-03-29T04:51:04.057 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.70935174Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-03-29T04:51:04.057 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.718961013Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-03-29T04:51:04.057 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.721715623Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-03-29T04:51:04.057 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.724304674Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-03-29T04:51:04.057 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.72770112Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-03-29T04:51:04.057 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.730517725Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-03-29T04:51:04.057 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.733509072Z level=info msg="Executing migration" id="create folder table" 2024-03-29T04:51:04.057 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.736123356Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-03-29T04:51:04.057 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.736699034Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-03-29T04:51:04.057 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.739552651Z level=info msg="Executing migration" id="Update folder title length" 2024-03-29T04:51:04.057 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.742229753Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-03-29T04:51:04.057 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=migrator t=2024-03-29T04:51:03.744701479Z level=info msg="migrations completed" performed=485 skipped=0 duration=705.458102ms 2024-03-29T04:51:04.057 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=sqlstore t=2024-03-29T04:51:03.745122727Z level=info msg="Created default organization" 2024-03-29T04:51:04.058 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=plugin.loader t=2024-03-29T04:51:03.799973084Z level=info msg="Plugin registered" pluginID=input 2024-03-29T04:51:04.058 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=plugin.loader t=2024-03-29T04:51:03.826767476Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-03-29T04:51:04.058 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=plugin.loader t=2024-03-29T04:51:03.826789503Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-03-29T04:51:04.058 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=secrets t=2024-03-29T04:51:03.826821837Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-03-29T04:51:04.058 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=query_data t=2024-03-29T04:51:03.830391488Z level=info msg="Query Service initialization" 2024-03-29T04:51:04.058 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=live.push_http t=2024-03-29T04:51:03.832871366Z level=info msg="Live Push Gateway initialization" 2024-03-29T04:51:04.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:04.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:04.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:04.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:04.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:04.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:04.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:04.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:04.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:04.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:04.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:04.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:04.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:04.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:04.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:04.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:04.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:04.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:04.880 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=infra.usagestats.collector t=2024-03-29T04:51:04.540634686Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-03-29T04:51:04.880 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=provisioning.datasources t=2024-03-29T04:51:04.540987103Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-03-29T04:51:04.880 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=provisioning.datasources t=2024-03-29T04:51:04.573462829Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-03-29T04:51:04.880 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=provisioning.datasources t=2024-03-29T04:51:04.605606414Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-03-29T04:51:04.880 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=provisioning.alerting t=2024-03-29T04:51:04.627117391Z level=info msg="starting to provision alerting" 2024-03-29T04:51:04.880 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=provisioning.alerting t=2024-03-29T04:51:04.627136975Z level=info msg="finished to provision alerting" 2024-03-29T04:51:04.880 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=grafanaStorageLogger t=2024-03-29T04:51:04.627332743Z level=info msg="storage starting" 2024-03-29T04:51:04.880 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=ngalert.state.manager t=2024-03-29T04:51:04.627339597Z level=info msg="Warming state cache for startup" 2024-03-29T04:51:04.880 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=http.server t=2024-03-29T04:51:04.629012961Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-03-29T04:51:04.881 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=sqlstore.transactions t=2024-03-29T04:51:04.638126373Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-03-29T04:51:04.881 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=ngalert.state.manager t=2024-03-29T04:51:04.638248152Z level=info msg="State cache has been initialized" states=0 duration=10.90747ms 2024-03-29T04:51:04.881 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=ticker t=2024-03-29T04:51:04.638303181Z level=info msg=starting first_tick=2024-03-29T04:51:10Z 2024-03-29T04:51:04.881 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=ngalert.multiorg.alertmanager t=2024-03-29T04:51:04.63830854Z level=info msg="starting MultiOrg Alertmanager" 2024-03-29T04:51:04.881 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=sqlstore.transactions t=2024-03-29T04:51:04.675186821Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-03-29T04:51:04.881 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=sqlstore.transactions t=2024-03-29T04:51:04.716916098Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-03-29T04:51:04.881 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:51:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[65719]: logger=sqlstore.transactions t=2024-03-29T04:51:04.728040332Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-03-29T04:51:05.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:05 smithi187 ceph-mon[33595]: Deploying daemon node-exporter.a on smithi181 2024-03-29T04:51:05.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:05 smithi187 ceph-mon[33595]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:05.383 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:05 smithi181 ceph-mon[41183]: Deploying daemon node-exporter.a on smithi181 2024-03-29T04:51:05.383 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:05 smithi181 ceph-mon[41183]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:05.384 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:05 smithi181 ceph-mon[29765]: Deploying daemon node-exporter.a on smithi181 2024-03-29T04:51:05.384 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:05 smithi181 ceph-mon[29765]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:07.034 INFO:teuthology.orchestra.run.smithi181.stdout:236223201293 2024-03-29T04:51:07.034 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-29T04:51:07.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:07 smithi187 ceph-mon[33595]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:07.449 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:07 smithi181 ceph-mon[41183]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:07.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:07 smithi181 ceph-mon[29765]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:08.807 INFO:teuthology.orchestra.run.smithi181.stdout:77309411367 2024-03-29T04:51:08.807 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-29T04:51:08.879 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:51:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:51:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:51:09.342 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:09 smithi181 ceph-mon[29765]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:09.342 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:09 smithi181 ceph-mon[41183]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:09.364 INFO:teuthology.orchestra.run.smithi181.stdout:34359738417 2024-03-29T04:51:09.365 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-29T04:51:09.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:09 smithi187 ceph-mon[33595]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:10.063 INFO:teuthology.orchestra.run.smithi181.stdout:201863462932 2024-03-29T04:51:10.063 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-29T04:51:11.379 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:11 smithi187 ceph-mon[33595]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:11.449 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:11 smithi181 ceph-mon[29765]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:11.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:11 smithi181 ceph-mon[41183]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:11.450 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:11 smithi181 bash[74522]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-03-29T04:51:12.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:12 smithi187 ceph-mon[33595]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:12.382 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:12 smithi181 ceph-mon[41183]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:12.382 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:12 smithi181 ceph-mon[29765]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:13.449 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:13 smithi181 bash[74522]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-03-29T04:51:13.915 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:13 smithi181 bash[74522]: Writing manifest to image destination 2024-03-29T04:51:14.199 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:13 smithi181 podman[74522]: 2024-03-29 04:51:13.943735012 +0000 UTC m=+6.162653878 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-03-29T04:51:14.441 INFO:teuthology.orchestra.run.smithi181.stdout:55834574893 2024-03-29T04:51:14.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-29T04:51:14.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:14 smithi181 ceph-mon[41183]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:14.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:14 smithi181 ceph-mon[29765]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:14.940 INFO:teuthology.orchestra.run.smithi181.stdout:167503724568 2024-03-29T04:51:14.941 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-29T04:51:15.078 INFO:teuthology.orchestra.run.smithi181.stdout:133143986206 2024-03-29T04:51:15.078 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-29T04:51:15.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:14 smithi187 ceph-mon[33595]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:15.537 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 podman[74522]: 2024-03-29 04:51:15.211822327 +0000 UTC m=+7.430741186 container create 2f1dfb6435e1cf391299df6eeb07a9611f6c5083b617dea12954d3ab08b91e0f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:51:15.537 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 podman[74522]: 2024-03-29 04:51:15.426147778 +0000 UTC m=+7.645066637 container init 2f1dfb6435e1cf391299df6eeb07a9611f6c5083b617dea12954d3ab08b91e0f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:51:15.538 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 podman[74522]: 2024-03-29 04:51:15.431620779 +0000 UTC m=+7.650539638 container start 2f1dfb6435e1cf391299df6eeb07a9611f6c5083b617dea12954d3ab08b91e0f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:51:15.538 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.435Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-03-29T04:51:15.538 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.435Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-03-29T04:51:15.538 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-03-29T04:51:15.538 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-03-29T04:51:15.538 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-03-29T04:51:15.538 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-03-29T04:51:15.538 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-03-29T04:51:15.538 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=arp 2024-03-29T04:51:15.538 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=bcache 2024-03-29T04:51:15.538 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=bonding 2024-03-29T04:51:15.538 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=btrfs 2024-03-29T04:51:15.539 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=conntrack 2024-03-29T04:51:15.539 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=cpu 2024-03-29T04:51:15.539 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-03-29T04:51:15.539 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=diskstats 2024-03-29T04:51:15.539 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=dmi 2024-03-29T04:51:15.539 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=edac 2024-03-29T04:51:15.539 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=entropy 2024-03-29T04:51:15.540 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-03-29T04:51:15.540 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=filefd 2024-03-29T04:51:15.540 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=filesystem 2024-03-29T04:51:15.540 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=hwmon 2024-03-29T04:51:15.540 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=infiniband 2024-03-29T04:51:15.540 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=ipvs 2024-03-29T04:51:15.540 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=loadavg 2024-03-29T04:51:15.540 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=mdadm 2024-03-29T04:51:15.540 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=meminfo 2024-03-29T04:51:15.540 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=netclass 2024-03-29T04:51:15.540 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=netdev 2024-03-29T04:51:15.540 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=netstat 2024-03-29T04:51:15.540 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=nfs 2024-03-29T04:51:15.541 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=nfsd 2024-03-29T04:51:15.541 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=nvme 2024-03-29T04:51:15.541 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=os 2024-03-29T04:51:15.541 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-03-29T04:51:15.541 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=pressure 2024-03-29T04:51:15.541 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=rapl 2024-03-29T04:51:15.541 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=schedstat 2024-03-29T04:51:15.541 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=selinux 2024-03-29T04:51:15.541 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=sockstat 2024-03-29T04:51:15.541 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=softnet 2024-03-29T04:51:15.541 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=stat 2024-03-29T04:51:15.541 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=tapestats 2024-03-29T04:51:15.541 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=textfile 2024-03-29T04:51:15.541 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-03-29T04:51:15.542 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=time 2024-03-29T04:51:15.542 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-03-29T04:51:15.542 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=uname 2024-03-29T04:51:15.542 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=vmstat 2024-03-29T04:51:15.542 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=xfs 2024-03-29T04:51:15.542 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.436Z caller=node_exporter.go:117 level=info collector=zfs 2024-03-29T04:51:15.542 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.437Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-03-29T04:51:15.542 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[75185]: ts=2024-03-29T04:51:15.437Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-03-29T04:51:15.542 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 bash[74522]: 2f1dfb6435e1cf391299df6eeb07a9611f6c5083b617dea12954d3ab08b91e0f 2024-03-29T04:51:15.848 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:51:15 smithi181 systemd[1]: Started Ceph node-exporter.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:51:15.859 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:15.859 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:15.863 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:15.902 INFO:teuthology.orchestra.run.smithi181.stdout:111669149732 2024-03-29T04:51:15.903 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-29T04:51:15.930 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:15.931 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:16.234 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:16.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:16 smithi181 ceph-mon[29765]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:16.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:16 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:16.699 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:16 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:16.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:16 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:16.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:16 smithi181 ceph-mon[29765]: Deploying daemon node-exporter.b on smithi187 2024-03-29T04:51:16.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:16 smithi181 ceph-mon[41183]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:16.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:16 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:16.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:16 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:16.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:16 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:16.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:16 smithi181 ceph-mon[41183]: Deploying daemon node-exporter.b on smithi187 2024-03-29T04:51:16.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:16 smithi187 ceph-mon[33595]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:16.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:16 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:16.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:16 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:16.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:16 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:16.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:16 smithi187 ceph-mon[33595]: Deploying daemon node-exporter.b on smithi187 2024-03-29T04:51:16.880 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:16 smithi187 bash[65993]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-03-29T04:51:17.380 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:17 smithi187 bash[65993]: Getting image source signatures 2024-03-29T04:51:17.380 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:17 smithi187 bash[65993]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-03-29T04:51:17.380 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:17 smithi187 bash[65993]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-03-29T04:51:17.380 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:17 smithi187 bash[65993]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-03-29T04:51:18.699 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:18 smithi181 ceph-mon[41183]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:18.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:18 smithi181 ceph-mon[29765]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:18.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:18 smithi187 ceph-mon[33595]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:18.880 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:51:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:51:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:51:19.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:19 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:19.948 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:19.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:19 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:19.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:19 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:20.879 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:20 smithi187 ceph-mon[33595]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:20.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:20 smithi181 ceph-mon[41183]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:20.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:20 smithi181 ceph-mon[29765]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:22.379 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:21 smithi187 bash[65993]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-03-29T04:51:22.380 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 bash[65993]: Writing manifest to image destination 2024-03-29T04:51:22.866 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-mon[33595]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:22.866 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 podman[65993]: 2024-03-29 04:51:22.610452195 +0000 UTC m=+6.003248712 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-03-29T04:51:22.866 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 podman[65993]: 2024-03-29 04:51:22.73402852 +0000 UTC m=+6.126825027 container create 9528215f6d2a52dbe07c06073d33d265e3882af0c35a23ea02b0ba4b3b5ced07 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:51:22.874 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:22 smithi181 ceph-mon[29765]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:22.874 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:22 smithi181 ceph-mon[41183]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:22.965 INFO:teuthology.orchestra.run.smithi181.stdout:34359738420 2024-03-29T04:51:23.180 INFO:teuthology.orchestra.run.smithi181.stdout:201863462935 2024-03-29T04:51:23.380 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 podman[65993]: 2024-03-29 04:51:22.969929 +0000 UTC m=+6.362725527 container init 9528215f6d2a52dbe07c06073d33d265e3882af0c35a23ea02b0ba4b3b5ced07 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:51:23.380 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 podman[65993]: 2024-03-29 04:51:22.976530774 +0000 UTC m=+6.369327280 container start 9528215f6d2a52dbe07c06073d33d265e3882af0c35a23ea02b0ba4b3b5ced07 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:51:23.380 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.980Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-03-29T04:51:23.380 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.980Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-03-29T04:51:23.381 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-03-29T04:51:23.381 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-03-29T04:51:23.381 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-03-29T04:51:23.381 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-03-29T04:51:23.381 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-03-29T04:51:23.381 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=node_exporter.go:117 level=info collector=arp 2024-03-29T04:51:23.381 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=node_exporter.go:117 level=info collector=bcache 2024-03-29T04:51:23.381 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=node_exporter.go:117 level=info collector=bonding 2024-03-29T04:51:23.381 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=node_exporter.go:117 level=info collector=btrfs 2024-03-29T04:51:23.381 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=node_exporter.go:117 level=info collector=conntrack 2024-03-29T04:51:23.381 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=node_exporter.go:117 level=info collector=cpu 2024-03-29T04:51:23.381 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-03-29T04:51:23.381 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=node_exporter.go:117 level=info collector=diskstats 2024-03-29T04:51:23.381 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=node_exporter.go:117 level=info collector=dmi 2024-03-29T04:51:23.381 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=node_exporter.go:117 level=info collector=edac 2024-03-29T04:51:23.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=node_exporter.go:117 level=info collector=entropy 2024-03-29T04:51:23.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-03-29T04:51:23.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=node_exporter.go:117 level=info collector=filefd 2024-03-29T04:51:23.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.981Z caller=node_exporter.go:117 level=info collector=filesystem 2024-03-29T04:51:23.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=hwmon 2024-03-29T04:51:23.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=infiniband 2024-03-29T04:51:23.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=ipvs 2024-03-29T04:51:23.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=loadavg 2024-03-29T04:51:23.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=mdadm 2024-03-29T04:51:23.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=meminfo 2024-03-29T04:51:23.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=netclass 2024-03-29T04:51:23.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=netdev 2024-03-29T04:51:23.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=netstat 2024-03-29T04:51:23.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=nfs 2024-03-29T04:51:23.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=nfsd 2024-03-29T04:51:23.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=nvme 2024-03-29T04:51:23.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=os 2024-03-29T04:51:23.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-03-29T04:51:23.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=pressure 2024-03-29T04:51:23.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=rapl 2024-03-29T04:51:23.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=schedstat 2024-03-29T04:51:23.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=selinux 2024-03-29T04:51:23.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=sockstat 2024-03-29T04:51:23.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=softnet 2024-03-29T04:51:23.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=stat 2024-03-29T04:51:23.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=tapestats 2024-03-29T04:51:23.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=textfile 2024-03-29T04:51:23.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-03-29T04:51:23.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=time 2024-03-29T04:51:23.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-03-29T04:51:23.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=uname 2024-03-29T04:51:23.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=vmstat 2024-03-29T04:51:23.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=xfs 2024-03-29T04:51:23.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=node_exporter.go:117 level=info collector=zfs 2024-03-29T04:51:23.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-03-29T04:51:23.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[66044]: ts=2024-03-29T04:51:22.982Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-03-29T04:51:23.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:23 smithi187 bash[65993]: 9528215f6d2a52dbe07c06073d33d265e3882af0c35a23ea02b0ba4b3b5ced07 2024-03-29T04:51:23.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:51:23 smithi187 systemd[1]: Started Ceph node-exporter.b for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:51:23.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:23 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1797997711' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:51:23.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:23 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:23.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:23 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:23.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:23 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2264179531' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:51:23.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:23 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:23.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:23 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:23.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:23 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:23.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:23 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:23.922 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:23 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1797997711' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:51:23.922 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:23 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:23.922 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:23 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:23.922 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:23 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2264179531' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:51:23.922 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:23 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:23.922 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:23 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:23.923 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:23 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:23.923 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:23 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:23.923 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:23 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1797997711' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:51:23.923 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:23 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:23.923 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:23 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:23.923 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:23 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2264179531' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:51:23.923 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:23 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:23.923 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:23 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:23.923 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:23 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:23.923 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:23 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:23.946 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462932 got 201863462935 for osd.6 2024-03-29T04:51:23.946 DEBUG:teuthology.parallel:result is None 2024-03-29T04:51:24.509 INFO:teuthology.orchestra.run.smithi181.stdout:167503724571 2024-03-29T04:51:24.590 INFO:teuthology.orchestra.run.smithi181.stdout:77309411371 2024-03-29T04:51:24.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:24 smithi187 ceph-mon[33595]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:24.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:24 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1679152925' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:51:24.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:24 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:24.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:24 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/638857936' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:51:24.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:24 smithi181 ceph-mon[41183]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:24.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:24 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1679152925' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:51:24.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:24 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:24.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:24 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/638857936' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:51:24.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:24 smithi181 ceph-mon[29765]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:24.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:24 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1679152925' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:51:24.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:24 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:24.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:24 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/638857936' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:51:25.279 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724568 got 167503724571 for osd.5 2024-03-29T04:51:25.280 DEBUG:teuthology.parallel:result is None 2024-03-29T04:51:25.683 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738417 got 34359738420 for osd.0 2024-03-29T04:51:25.683 DEBUG:teuthology.parallel:result is None 2024-03-29T04:51:26.271 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:26.629 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:26 smithi187 ceph-mon[33595]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:26.655 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:26 smithi181 ceph-mon[41183]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:26.655 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:26 smithi181 ceph-mon[29765]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:27.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:27 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:27.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:27 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:27.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:27 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:27.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:27 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:28.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:27 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:28.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:27 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:28.130 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411367 got 77309411371 for osd.2 2024-03-29T04:51:28.131 DEBUG:teuthology.parallel:result is None 2024-03-29T04:51:28.787 INFO:teuthology.orchestra.run.smithi181.stdout:236223201298 2024-03-29T04:51:28.843 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:51:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:51:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:51:28.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:28 smithi181 ceph-mon[29765]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:28.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:28 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1776687741' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:51:28.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:28 smithi181 ceph-mon[41183]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:28.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:28 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1776687741' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:51:29.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:28 smithi187 ceph-mon[33595]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:29.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:28 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1776687741' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:51:29.634 INFO:teuthology.orchestra.run.smithi181.stdout:133143986209 2024-03-29T04:51:29.945 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:29 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1511129137' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:51:29.945 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:29 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1511129137' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:51:30.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:29 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1511129137' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:51:30.346 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986206 got 133143986209 for osd.4 2024-03-29T04:51:30.346 DEBUG:teuthology.parallel:result is None 2024-03-29T04:51:30.983 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201293 got 236223201298 for osd.7 2024-03-29T04:51:30.983 DEBUG:teuthology.parallel:result is None 2024-03-29T04:51:31.128 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:30 smithi181 ceph-mon[41183]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:31.128 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:30 smithi181 ceph-mon[29765]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:31.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:30 smithi187 ceph-mon[33595]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:31.898 INFO:teuthology.orchestra.run.smithi181.stdout:55834574897 2024-03-29T04:51:32.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:31 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2133780564' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:51:32.177 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:31 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2133780564' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:51:32.177 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:31 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2133780564' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:51:32.460 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574893 got 55834574897 for osd.1 2024-03-29T04:51:32.460 DEBUG:teuthology.parallel:result is None 2024-03-29T04:51:33.022 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:32 smithi181 ceph-mon[29765]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:33.022 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:32 smithi181 ceph-mon[41183]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:33.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:32 smithi187 ceph-mon[33595]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:33.244 INFO:teuthology.orchestra.run.smithi181.stdout:111669149736 2024-03-29T04:51:33.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:33 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2844620319' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:51:33.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:33 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:33.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:33 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2844620319' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:51:33.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:33 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:34.001 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149732 got 111669149736 for osd.3 2024-03-29T04:51:34.001 DEBUG:teuthology.parallel:result is None 2024-03-29T04:51:34.001 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-29T04:51:34.001 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T04:51:34.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:33 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2844620319' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:51:34.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:33 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:34.281 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:34.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:34 smithi181 ceph-mon[29765]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:34.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:34 smithi181 ceph-mon[41183]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:35.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:34 smithi187 ceph-mon[33595]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:36.119 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:51:36.121 INFO:teuthology.orchestra.run.smithi181.stderr:dumped all 2024-03-29T04:51:37.087 INFO:teuthology.orchestra.run.smithi181.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2024-03-29T04:51:35.596428+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385560,"kb_used_data":6692,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575656,"statfs":{"total":767960285184,"available":765517471744,"internally_reserved":0,"allocated":6852608,"data_stored":3896616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002084"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:41.657784+0000","last_change":"2024-03-29T04:50:39.046369+0000","last_active":"2024-03-29T04:50:41.657784+0000","last_peered":"2024-03-29T04:50:41.657784+0000","last_clean":"2024-03-29T04:50:41.657784+0000","last_became_active":"2024-03-29T04:50:39.046086+0000","last_became_peered":"2024-03-29T04:50:39.046086+0000","last_unstale":"2024-03-29T04:50:41.657784+0000","last_undegraded":"2024-03-29T04:50:41.657784+0000","last_fullsized":"2024-03-29T04:50:41.657784+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:26:33.515943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:40.875134+0000","last_change":"2024-03-29T04:50:39.220266+0000","last_active":"2024-03-29T04:50:40.875134+0000","last_peered":"2024-03-29T04:50:40.875134+0000","last_clean":"2024-03-29T04:50:40.875134+0000","last_became_active":"2024-03-29T04:50:39.219919+0000","last_became_peered":"2024-03-29T04:50:39.219919+0000","last_unstale":"2024-03-29T04:50:40.875134+0000","last_undegraded":"2024-03-29T04:50:40.875134+0000","last_fullsized":"2024-03-29T04:50:40.875134+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:16:10.835704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:40.053614+0000","last_change":"2024-03-29T04:50:39.219980+0000","last_active":"2024-03-29T04:50:40.053614+0000","last_peered":"2024-03-29T04:50:40.053614+0000","last_clean":"2024-03-29T04:50:40.053614+0000","last_became_active":"2024-03-29T04:50:39.219627+0000","last_became_peered":"2024-03-29T04:50:39.219627+0000","last_unstale":"2024-03-29T04:50:40.053614+0000","last_undegraded":"2024-03-29T04:50:40.053614+0000","last_fullsized":"2024-03-29T04:50:40.053614+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:56:32.992193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:40.053658+0000","last_change":"2024-03-29T04:50:39.149486+0000","last_active":"2024-03-29T04:50:40.053658+0000","last_peered":"2024-03-29T04:50:40.053658+0000","last_clean":"2024-03-29T04:50:40.053658+0000","last_became_active":"2024-03-29T04:50:39.149236+0000","last_became_peered":"2024-03-29T04:50:39.149236+0000","last_unstale":"2024-03-29T04:50:40.053658+0000","last_undegraded":"2024-03-29T04:50:40.053658+0000","last_fullsized":"2024-03-29T04:50:40.053658+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:14:50.050422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:40.053616+0000","last_change":"2024-03-29T04:50:05.432965+0000","last_active":"2024-03-29T04:50:40.053616+0000","last_peered":"2024-03-29T04:50:40.053616+0000","last_clean":"2024-03-29T04:50:40.053616+0000","last_became_active":"2024-03-29T04:50:05.432530+0000","last_became_peered":"2024-03-29T04:50:05.432530+0000","last_unstale":"2024-03-29T04:50:40.053616+0000","last_undegraded":"2024-03-29T04:50:40.053616+0000","last_fullsized":"2024-03-29T04:50:40.053616+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:48:01.601528+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:48:01.601528+0000","last_clean_scrub_stamp":"2024-03-29T04:48:01.601528+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:56:20.936763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:41.595875+0000","last_change":"2024-03-29T04:50:39.045035+0000","last_active":"2024-03-29T04:50:41.595875+0000","last_peered":"2024-03-29T04:50:41.595875+0000","last_clean":"2024-03-29T04:50:41.595875+0000","last_became_active":"2024-03-29T04:50:39.044724+0000","last_became_peered":"2024-03-29T04:50:39.044724+0000","last_unstale":"2024-03-29T04:50:41.595875+0000","last_undegraded":"2024-03-29T04:50:41.595875+0000","last_fullsized":"2024-03-29T04:50:41.595875+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:44:08.632467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:40.049669+0000","last_change":"2024-03-29T04:50:39.246218+0000","last_active":"2024-03-29T04:50:40.049669+0000","last_peered":"2024-03-29T04:50:40.049669+0000","last_clean":"2024-03-29T04:50:40.049669+0000","last_became_active":"2024-03-29T04:50:39.245880+0000","last_became_peered":"2024-03-29T04:50:39.245880+0000","last_unstale":"2024-03-29T04:50:40.049669+0000","last_undegraded":"2024-03-29T04:50:40.049669+0000","last_fullsized":"2024-03-29T04:50:40.049669+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:17:07.467340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:41.595879+0000","last_change":"2024-03-29T04:50:39.044530+0000","last_active":"2024-03-29T04:50:41.595879+0000","last_peered":"2024-03-29T04:50:41.595879+0000","last_clean":"2024-03-29T04:50:41.595879+0000","last_became_active":"2024-03-29T04:50:39.044136+0000","last_became_peered":"2024-03-29T04:50:39.044136+0000","last_unstale":"2024-03-29T04:50:41.595879+0000","last_undegraded":"2024-03-29T04:50:41.595879+0000","last_fullsized":"2024-03-29T04:50:41.595879+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:32:12.577651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:40.875149+0000","last_change":"2024-03-29T04:50:39.149460+0000","last_active":"2024-03-29T04:50:40.875149+0000","last_peered":"2024-03-29T04:50:40.875149+0000","last_clean":"2024-03-29T04:50:40.875149+0000","last_became_active":"2024-03-29T04:50:39.149152+0000","last_became_peered":"2024-03-29T04:50:39.149152+0000","last_unstale":"2024-03-29T04:50:40.875149+0000","last_undegraded":"2024-03-29T04:50:40.875149+0000","last_fullsized":"2024-03-29T04:50:40.875149+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:05:37.697946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201300,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:51:04 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.55000000000000004}]},{"osd":1,"last update":"Fri Mar 29 04:51:04 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.57799999999999996}]},{"osd":2,"last update":"Fri Mar 29 04:51:04 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.53800000000000003}]},{"osd":3,"last update":"Fri Mar 29 04:51:04 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.44500000000000001}]},{"osd":4,"last update":"Fri Mar 29 04:51:04 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.50900000000000001}]},{"osd":5,"last update":"Fri Mar 29 04:51:04 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.625}]},{"osd":6,"last update":"Fri Mar 29 04:51:04 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.64200000000000002}]}]},{"osd":6,"up_from":47,"seq":201863462938,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1224704,"data_stored":855822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:50:36 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.372},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.58499999999999996}]},{"osd":1,"last update":"Fri Mar 29 04:50:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.498}]},{"osd":2,"last update":"Fri Mar 29 04:50:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.438},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.51500000000000001}]},{"osd":3,"last update":"Fri Mar 29 04:50:36 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.53400000000000003}]},{"osd":4,"last update":"Fri Mar 29 04:50:36 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.57499999999999996}]},{"osd":5,"last update":"Fri Mar 29 04:50:36 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.47899999999999998}]},{"osd":7,"last update":"Fri Mar 29 04:51:06 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.54700000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":265512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:50:40 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.34799999999999998,"15min":0.33600000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.55100000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.443,"5min":0.374,"15min":0.34699999999999998},"min":{"1min":0.32200000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.45300000000000001}]},{"osd":2,"last update":"Fri Mar 29 04:51:02 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.39800000000000002,"15min":0.38400000000000001},"min":{"1min":0.27300000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.45000000000000001,"15min":0.433},"min":{"1min":0.27700000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.58899999999999997}]},{"osd":3,"last update":"Fri Mar 29 04:51:33 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.45100000000000001,"15min":0.45500000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.57699999999999996,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.436,"15min":0.44500000000000001},"min":{"1min":0.26500000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.60699999999999998,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.55200000000000005}]},{"osd":4,"last update":"Fri Mar 29 04:50:48 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.56999999999999995,"15min":0.57299999999999995},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.48199999999999998,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.76600000000000001,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.56699999999999995}]},{"osd":5,"last update":"Fri Mar 29 04:51:16 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57199999999999995,"15min":0.57099999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.57999999999999996,"15min":0.58199999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.68200000000000005}]},{"osd":6,"last update":"Fri Mar 29 04:50:37 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.623}]},{"osd":7,"last update":"Fri Mar 29 04:51:02 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.70599999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1228800,"data_stored":856145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Mar 29 04:50:42 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.378,"15min":0.36599999999999999},"min":{"1min":0.29299999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.52600000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.433},{"interface":"front","average":{"1min":0.375,"5min":0.38300000000000001,"15min":0.38200000000000001},"min":{"1min":0.26400000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.501,"5min":0.621,"15min":0.621},"last":0.41899999999999998}]},{"osd":2,"last update":"Fri Mar 29 04:51:03 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.40799999999999997,"15min":0.40300000000000002},"min":{"1min":0.312,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.61499999999999999,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.38100000000000001}]},{"osd":3,"last update":"Fri Mar 29 04:51:30 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.44900000000000001,"15min":0.45400000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.57799999999999996,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.42899999999999999,"15min":0.436},"min":{"1min":0.33000000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.46899999999999997}]},{"osd":4,"last update":"Fri Mar 29 04:50:52 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.58099999999999996,"15min":0.58799999999999997},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.71599999999999997,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.77700000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56599999999999995,"15min":0.56999999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.70299999999999996,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.74299999999999999}]},{"osd":5,"last update":"Fri Mar 29 04:51:17 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58199999999999996,"15min":0.58099999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.56000000000000005,"15min":0.55400000000000005},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.75800000000000001}]},{"osd":6,"last update":"Fri Mar 29 04:50:39 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.73199999999999998}]},{"osd":7,"last update":"Fri Mar 29 04:51:03 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.70199999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:51:02 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36299999999999999,"15min":0.35199999999999998},"min":{"1min":0.27000000000000002,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.52000000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.307},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.379,"15min":0.36599999999999999},"min":{"1min":0.255,"5min":0.215,"15min":0.215},"max":{"1min":0.50600000000000001,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.32000000000000001}]},{"osd":1,"last update":"Fri Mar 29 04:51:02 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.379},"min":{"1min":0.24199999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.435},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.40500000000000003,"15min":0.39000000000000001},"min":{"1min":0.31900000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.57199999999999995,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.27900000000000003}]},{"osd":3,"last update":"Fri Mar 29 04:51:29 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.495,"15min":0.52100000000000002},"min":{"1min":0.30499999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.53700000000000003,"5min":1.232,"15min":1.232},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.46800000000000003,"15min":0.48499999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.51300000000000001,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.36399999999999999}]},{"osd":4,"last update":"Fri Mar 29 04:50:49 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.60099999999999998,"15min":0.59599999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.63,"15min":0.625},"min":{"1min":0.46300000000000002,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.79700000000000004}]},{"osd":5,"last update":"Fri Mar 29 04:51:14 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.59599999999999997,"15min":0.60399999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.79300000000000004,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.63,"15min":0.64000000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.72599999999999998}]},{"osd":6,"last update":"Fri Mar 29 04:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.754},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.77600000000000002}]},{"osd":7,"last update":"Fri Mar 29 04:51:03 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.60399999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149737,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:51:31 2024","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.39400000000000002,"15min":0.40899999999999997},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.505,"5min":0.53600000000000003,"15min":0.53600000000000003},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.34699999999999998,"5min":0.42499999999999999,"15min":0.435},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.48999999999999999,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.34100000000000003}]},{"osd":1,"last update":"Fri Mar 29 04:51:31 2024","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":0.39400000000000002,"15min":0.40400000000000003},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.52400000000000002,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.374},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.44900000000000001,"15min":0.45800000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.35999999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:51:31 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.45700000000000002,"15min":0.46300000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.54900000000000004,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.45900000000000002,"15min":0.48699999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.53000000000000003,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.38700000000000001}]},{"osd":4,"last update":"Fri Mar 29 04:50:49 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.61899999999999999,"15min":0.623},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.58599999999999997,"15min":0.58899999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.71399999999999997}]},{"osd":5,"last update":"Fri Mar 29 04:51:18 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.60999999999999999,"15min":0.61799999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.80900000000000005,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.61599999999999999,"15min":0.61899999999999999},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":1.042,"5min":1.042,"15min":1.042},"last":0.55300000000000005}]},{"osd":6,"last update":"Fri Mar 29 04:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.628}]},{"osd":7,"last update":"Fri Mar 29 04:51:05 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.754},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.56200000000000006}]}]},{"osd":4,"up_from":31,"seq":133143986211,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":630784,"data_stored":265512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:50:53 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.56499999999999995,"15min":0.56999999999999995},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70999999999999996,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.40699999999999997}]},{"osd":1,"last update":"Fri Mar 29 04:50:53 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.53900000000000003,"15min":0.53800000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.76800000000000002,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.54800000000000004,"15min":0.55200000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.69399999999999995,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.54200000000000004}]},{"osd":2,"last update":"Fri Mar 29 04:50:53 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.56000000000000005,"15min":0.56200000000000006},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.72199999999999998,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54400000000000004,"15min":0.54300000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76500000000000001,"5min":0.877,"15min":0.877},"last":0.46100000000000002}]},{"osd":3,"last update":"Fri Mar 29 04:50:53 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.39400000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.56999999999999995,"15min":0.57199999999999995},"min":{"1min":0.38300000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.79900000000000004,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.47199999999999998}]},{"osd":5,"last update":"Fri Mar 29 04:51:13 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.41999999999999998,"15min":0.41599999999999998},"min":{"1min":0.25600000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.47999999999999998,"15min":0.47399999999999998},"min":{"1min":0.34300000000000003,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.56000000000000005}]},{"osd":6,"last update":"Fri Mar 29 04:50:36 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.372}]},{"osd":7,"last update":"Fri Mar 29 04:51:07 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.439},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.38800000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724574,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":634880,"data_stored":265719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:51:20 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.54100000000000004,"15min":0.54500000000000004},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.378,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.68700000000000006,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.44400000000000001}]},{"osd":1,"last update":"Fri Mar 29 04:51:20 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.54700000000000004,"15min":0.54900000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.442,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.373}]},{"osd":2,"last update":"Fri Mar 29 04:51:20 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.58099999999999996,"15min":0.58499999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.71999999999999997,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.40500000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.42399999999999999}]},{"osd":3,"last update":"Fri Mar 29 04:51:20 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.54500000000000004,"15min":0.54400000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.75,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.58099999999999996,"15min":0.58499999999999996},"min":{"1min":0.44,"5min":0.37,"15min":0.37},"max":{"1min":0.69699999999999995,"5min":0.753,"15min":0.753},"last":0.41099999999999998}]},{"osd":4,"last update":"Fri Mar 29 04:51:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.40999999999999998,"15min":0.39100000000000001},"min":{"1min":0.33200000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.075,"5min":1.075,"15min":1.075},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.432,"15min":0.41899999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.34000000000000002}]},{"osd":6,"last update":"Fri Mar 29 04:50:39 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.434}]},{"osd":7,"last update":"Fri Mar 29 04:51:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.115,"5min":1.115,"15min":1.115},"last":0.40400000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T04:51:37.089 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T04:51:37.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:36 smithi187 ceph-mon[33595]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:37.133 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:36 smithi181 ceph-mon[29765]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:37.133 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:36 smithi181 ceph-mon[41183]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:37.483 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:38.117 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:37 smithi181 ceph-mon[29765]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:51:38.118 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:37 smithi181 ceph-mon[41183]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:51:38.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:37 smithi187 ceph-mon[33595]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:51:38.833 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:51:38 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:51:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:51:38.880 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:38 smithi181 ceph-mon[29765]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:38.880 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:38 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:38.880 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:38 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:39.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:38 smithi187 ceph-mon[33595]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:39.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:38 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:39.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:38 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:39.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:38 smithi181 ceph-mon[41183]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:39.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:38 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:39.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:38 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:40.079 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:51:40.080 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:39 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:40.080 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:39 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:51:40.080 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:39 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:40.080 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:39 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:40.081 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:39 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:51:40.081 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:39 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:40.082 INFO:teuthology.orchestra.run.smithi181.stderr:dumped all 2024-03-29T04:51:40.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:39 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:40.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:39 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:51:40.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:39 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:40.748 INFO:teuthology.orchestra.run.smithi181.stdout:{"pg_ready":true,"pg_map":{"version":40,"stamp":"2024-03-29T04:51:39.597088+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385560,"kb_used_data":6692,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575656,"statfs":{"total":767960285184,"available":765517471744,"internally_reserved":0,"allocated":6852608,"data_stored":3896616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001970"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:41.657784+0000","last_change":"2024-03-29T04:50:39.046369+0000","last_active":"2024-03-29T04:50:41.657784+0000","last_peered":"2024-03-29T04:50:41.657784+0000","last_clean":"2024-03-29T04:50:41.657784+0000","last_became_active":"2024-03-29T04:50:39.046086+0000","last_became_peered":"2024-03-29T04:50:39.046086+0000","last_unstale":"2024-03-29T04:50:41.657784+0000","last_undegraded":"2024-03-29T04:50:41.657784+0000","last_fullsized":"2024-03-29T04:50:41.657784+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:26:33.515943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:40.875134+0000","last_change":"2024-03-29T04:50:39.220266+0000","last_active":"2024-03-29T04:50:40.875134+0000","last_peered":"2024-03-29T04:50:40.875134+0000","last_clean":"2024-03-29T04:50:40.875134+0000","last_became_active":"2024-03-29T04:50:39.219919+0000","last_became_peered":"2024-03-29T04:50:39.219919+0000","last_unstale":"2024-03-29T04:50:40.875134+0000","last_undegraded":"2024-03-29T04:50:40.875134+0000","last_fullsized":"2024-03-29T04:50:40.875134+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:16:10.835704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:40.053614+0000","last_change":"2024-03-29T04:50:39.219980+0000","last_active":"2024-03-29T04:50:40.053614+0000","last_peered":"2024-03-29T04:50:40.053614+0000","last_clean":"2024-03-29T04:50:40.053614+0000","last_became_active":"2024-03-29T04:50:39.219627+0000","last_became_peered":"2024-03-29T04:50:39.219627+0000","last_unstale":"2024-03-29T04:50:40.053614+0000","last_undegraded":"2024-03-29T04:50:40.053614+0000","last_fullsized":"2024-03-29T04:50:40.053614+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:56:32.992193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:40.053658+0000","last_change":"2024-03-29T04:50:39.149486+0000","last_active":"2024-03-29T04:50:40.053658+0000","last_peered":"2024-03-29T04:50:40.053658+0000","last_clean":"2024-03-29T04:50:40.053658+0000","last_became_active":"2024-03-29T04:50:39.149236+0000","last_became_peered":"2024-03-29T04:50:39.149236+0000","last_unstale":"2024-03-29T04:50:40.053658+0000","last_undegraded":"2024-03-29T04:50:40.053658+0000","last_fullsized":"2024-03-29T04:50:40.053658+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:14:50.050422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:40.053616+0000","last_change":"2024-03-29T04:50:05.432965+0000","last_active":"2024-03-29T04:50:40.053616+0000","last_peered":"2024-03-29T04:50:40.053616+0000","last_clean":"2024-03-29T04:50:40.053616+0000","last_became_active":"2024-03-29T04:50:05.432530+0000","last_became_peered":"2024-03-29T04:50:05.432530+0000","last_unstale":"2024-03-29T04:50:40.053616+0000","last_undegraded":"2024-03-29T04:50:40.053616+0000","last_fullsized":"2024-03-29T04:50:40.053616+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:48:01.601528+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:48:01.601528+0000","last_clean_scrub_stamp":"2024-03-29T04:48:01.601528+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:56:20.936763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:41.595875+0000","last_change":"2024-03-29T04:50:39.045035+0000","last_active":"2024-03-29T04:50:41.595875+0000","last_peered":"2024-03-29T04:50:41.595875+0000","last_clean":"2024-03-29T04:50:41.595875+0000","last_became_active":"2024-03-29T04:50:39.044724+0000","last_became_peered":"2024-03-29T04:50:39.044724+0000","last_unstale":"2024-03-29T04:50:41.595875+0000","last_undegraded":"2024-03-29T04:50:41.595875+0000","last_fullsized":"2024-03-29T04:50:41.595875+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:44:08.632467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:40.049669+0000","last_change":"2024-03-29T04:50:39.246218+0000","last_active":"2024-03-29T04:50:40.049669+0000","last_peered":"2024-03-29T04:50:40.049669+0000","last_clean":"2024-03-29T04:50:40.049669+0000","last_became_active":"2024-03-29T04:50:39.245880+0000","last_became_peered":"2024-03-29T04:50:39.245880+0000","last_unstale":"2024-03-29T04:50:40.049669+0000","last_undegraded":"2024-03-29T04:50:40.049669+0000","last_fullsized":"2024-03-29T04:50:40.049669+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:17:07.467340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:41.595879+0000","last_change":"2024-03-29T04:50:39.044530+0000","last_active":"2024-03-29T04:50:41.595879+0000","last_peered":"2024-03-29T04:50:41.595879+0000","last_clean":"2024-03-29T04:50:41.595879+0000","last_became_active":"2024-03-29T04:50:39.044136+0000","last_became_peered":"2024-03-29T04:50:39.044136+0000","last_unstale":"2024-03-29T04:50:41.595879+0000","last_undegraded":"2024-03-29T04:50:41.595879+0000","last_fullsized":"2024-03-29T04:50:41.595879+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:32:12.577651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:50:40.875149+0000","last_change":"2024-03-29T04:50:39.149460+0000","last_active":"2024-03-29T04:50:40.875149+0000","last_peered":"2024-03-29T04:50:40.875149+0000","last_clean":"2024-03-29T04:50:40.875149+0000","last_became_active":"2024-03-29T04:50:39.149152+0000","last_became_peered":"2024-03-29T04:50:39.149152+0000","last_unstale":"2024-03-29T04:50:40.875149+0000","last_undegraded":"2024-03-29T04:50:40.875149+0000","last_fullsized":"2024-03-29T04:50:40.875149+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:05:37.697946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201301,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:51:04 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.623},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.45200000000000001}]},{"osd":1,"last update":"Fri Mar 29 04:51:04 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.50600000000000001}]},{"osd":2,"last update":"Fri Mar 29 04:51:04 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.47099999999999997}]},{"osd":3,"last update":"Fri Mar 29 04:51:04 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.433},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.53300000000000003}]},{"osd":4,"last update":"Fri Mar 29 04:51:04 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.55000000000000004}]},{"osd":5,"last update":"Fri Mar 29 04:51:04 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.56599999999999995}]},{"osd":6,"last update":"Fri Mar 29 04:51:04 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.58599999999999997}]}]},{"osd":6,"up_from":47,"seq":201863462939,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1196,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1224704,"data_stored":855822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:50:36 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.70399999999999996}]},{"osd":1,"last update":"Fri Mar 29 04:50:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.73799999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:50:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.65600000000000003}]},{"osd":3,"last update":"Fri Mar 29 04:50:36 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.68899999999999995}]},{"osd":4,"last update":"Fri Mar 29 04:50:36 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.52800000000000002}]},{"osd":5,"last update":"Fri Mar 29 04:50:36 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.56899999999999995}]},{"osd":7,"last update":"Fri Mar 29 04:51:06 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.48899999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":265512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:50:40 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.34799999999999998,"15min":0.33600000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.55100000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.443,"5min":0.374,"15min":0.34699999999999998},"min":{"1min":0.32200000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.39700000000000002}]},{"osd":2,"last update":"Fri Mar 29 04:51:02 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.39800000000000002,"15min":0.38400000000000001},"min":{"1min":0.27300000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.45000000000000001,"15min":0.433},"min":{"1min":0.27700000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.439}]},{"osd":3,"last update":"Fri Mar 29 04:51:33 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.45100000000000001,"15min":0.45500000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.57699999999999996,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.436,"15min":0.44500000000000001},"min":{"1min":0.26500000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.60699999999999998,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.36699999999999999}]},{"osd":4,"last update":"Fri Mar 29 04:50:48 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.56999999999999995,"15min":0.57299999999999995},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.48199999999999998,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.76600000000000001,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.5}]},{"osd":5,"last update":"Fri Mar 29 04:51:16 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57199999999999995,"15min":0.57099999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.57999999999999996,"15min":0.58199999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.69999999999999996}]},{"osd":6,"last update":"Fri Mar 29 04:51:38 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57999999999999996,"15min":0.58099999999999996},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.60099999999999998,"15min":0.60699999999999998},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.73799999999999999,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.67900000000000005}]},{"osd":7,"last update":"Fri Mar 29 04:51:02 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.48899999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738424,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1228800,"data_stored":856145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Mar 29 04:50:42 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.378,"15min":0.36599999999999999},"min":{"1min":0.29299999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.52600000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.31},{"interface":"front","average":{"1min":0.375,"5min":0.38300000000000001,"15min":0.38200000000000001},"min":{"1min":0.26400000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.501,"5min":0.621,"15min":0.621},"last":0.40200000000000002}]},{"osd":2,"last update":"Fri Mar 29 04:51:03 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.40799999999999997,"15min":0.40300000000000002},"min":{"1min":0.312,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.61499999999999999,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.34599999999999997}]},{"osd":3,"last update":"Fri Mar 29 04:51:30 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.44900000000000001,"15min":0.45400000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.57799999999999996,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.42899999999999999,"15min":0.436},"min":{"1min":0.33000000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.377}]},{"osd":4,"last update":"Fri Mar 29 04:50:52 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.58099999999999996,"15min":0.58799999999999997},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.71599999999999997,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56599999999999995,"15min":0.56999999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.70299999999999996,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.56899999999999995}]},{"osd":5,"last update":"Fri Mar 29 04:51:17 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58199999999999996,"15min":0.58099999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.56000000000000005,"15min":0.55400000000000005},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.58799999999999997}]},{"osd":6,"last update":"Fri Mar 29 04:50:39 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.50800000000000001}]},{"osd":7,"last update":"Fri Mar 29 04:51:03 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.60599999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411375,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:51:02 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36299999999999999,"15min":0.35199999999999998},"min":{"1min":0.27000000000000002,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.52000000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.379,"15min":0.36599999999999999},"min":{"1min":0.255,"5min":0.215,"15min":0.215},"max":{"1min":0.50600000000000001,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.35599999999999998}]},{"osd":1,"last update":"Fri Mar 29 04:51:02 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.379},"min":{"1min":0.24199999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.40500000000000003,"15min":0.39000000000000001},"min":{"1min":0.31900000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.57199999999999995,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.38300000000000001}]},{"osd":3,"last update":"Fri Mar 29 04:51:29 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.495,"15min":0.52100000000000002},"min":{"1min":0.30499999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.53700000000000003,"5min":1.232,"15min":1.232},"last":0.434},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.46800000000000003,"15min":0.48499999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.51300000000000001,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.375}]},{"osd":4,"last update":"Fri Mar 29 04:50:49 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.60099999999999998,"15min":0.59599999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.753},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.63,"15min":0.625},"min":{"1min":0.46300000000000002,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.76000000000000001}]},{"osd":5,"last update":"Fri Mar 29 04:51:14 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.59599999999999997,"15min":0.60399999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.79300000000000004,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.63,"15min":0.64000000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.78100000000000003}]},{"osd":6,"last update":"Fri Mar 29 04:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.68999999999999995}]},{"osd":7,"last update":"Fri Mar 29 04:51:03 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.80400000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149738,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:51:31 2024","interfaces":[{"interface":"back","average":{"1min":0.33200000000000002,"5min":0.39400000000000002,"15min":0.40899999999999997},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.505,"5min":0.53600000000000003,"15min":0.53600000000000003},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.34699999999999998,"5min":0.42499999999999999,"15min":0.435},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.48999999999999999,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.38100000000000001}]},{"osd":1,"last update":"Fri Mar 29 04:51:31 2024","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":0.39400000000000002,"15min":0.40400000000000003},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.52400000000000002,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.44900000000000001,"15min":0.45800000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.30199999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:51:31 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.45700000000000002,"15min":0.46300000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.54900000000000004,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.35999999999999999,"5min":0.45900000000000002,"15min":0.48699999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.53000000000000003,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.40699999999999997}]},{"osd":4,"last update":"Fri Mar 29 04:50:49 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.61899999999999999,"15min":0.623},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.58599999999999997,"15min":0.58899999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.69799999999999995}]},{"osd":5,"last update":"Fri Mar 29 04:51:18 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.60999999999999999,"15min":0.61799999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.80900000000000005,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.61599999999999999,"15min":0.61899999999999999},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":1.042,"5min":1.042,"15min":1.042},"last":0.57699999999999996}]},{"osd":6,"last update":"Fri Mar 29 04:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.76000000000000001}]},{"osd":7,"last update":"Fri Mar 29 04:51:05 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.67500000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986212,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":630784,"data_stored":265512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:50:53 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.56499999999999995,"15min":0.56999999999999995},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70999999999999996,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.47199999999999998}]},{"osd":1,"last update":"Fri Mar 29 04:50:53 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.53900000000000003,"15min":0.53800000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.76800000000000002,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.54800000000000004,"15min":0.55200000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.69399999999999995,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.621}]},{"osd":2,"last update":"Fri Mar 29 04:50:53 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.56000000000000005,"15min":0.56200000000000006},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.72199999999999998,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.5},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54400000000000004,"15min":0.54300000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76500000000000001,"5min":0.877,"15min":0.877},"last":0.41299999999999998}]},{"osd":3,"last update":"Fri Mar 29 04:50:53 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.39400000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.56999999999999995,"15min":0.57199999999999995},"min":{"1min":0.38300000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.79900000000000004,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.436}]},{"osd":5,"last update":"Fri Mar 29 04:51:13 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.41999999999999998,"15min":0.41599999999999998},"min":{"1min":0.25600000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.47999999999999998,"15min":0.47399999999999998},"min":{"1min":0.34300000000000003,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.60599999999999998}]},{"osd":6,"last update":"Fri Mar 29 04:51:37 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.47699999999999998,"15min":0.47499999999999998},"min":{"1min":0.29699999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.68400000000000005,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.47499999999999998,"15min":0.47299999999999998},"min":{"1min":0.34799999999999998,"5min":0.316,"15min":0.316},"max":{"1min":0.71699999999999997,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.55300000000000005}]},{"osd":7,"last update":"Fri Mar 29 04:51:07 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.57899999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724575,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":634880,"data_stored":265719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:51:20 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.54100000000000004,"15min":0.54500000000000004},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.378,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.68700000000000006,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.48299999999999998}]},{"osd":1,"last update":"Fri Mar 29 04:51:20 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.54700000000000004,"15min":0.54900000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.442,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.64500000000000002}]},{"osd":2,"last update":"Fri Mar 29 04:51:20 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.58099999999999996,"15min":0.58499999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.71999999999999997,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.40500000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.61099999999999999}]},{"osd":3,"last update":"Fri Mar 29 04:51:20 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.54500000000000004,"15min":0.54400000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.75,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.58099999999999996,"15min":0.58499999999999996},"min":{"1min":0.44,"5min":0.37,"15min":0.37},"max":{"1min":0.69699999999999995,"5min":0.753,"15min":0.753},"last":0.625}]},{"osd":4,"last update":"Fri Mar 29 04:51:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.40999999999999998,"15min":0.39100000000000001},"min":{"1min":0.33200000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.075,"5min":1.075,"15min":1.075},"last":0.38},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.432,"15min":0.41899999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.495}]},{"osd":6,"last update":"Fri Mar 29 04:50:39 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.40000000000000002}]},{"osd":7,"last update":"Fri Mar 29 04:51:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.115,"5min":1.115,"15min":1.115},"last":0.44}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T04:51:40.750 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-29T04:51:40.750 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-29T04:51:40.750 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-29T04:51:40.750 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:51:40.991 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:40 smithi181 ceph-mon[29765]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:40.991 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:40 smithi181 ceph-mon[29765]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:51:40.991 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:40 smithi181 ceph-mon[29765]: Reconfiguring daemon alertmanager.a on smithi181 2024-03-29T04:51:40.991 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:40 smithi181 ceph-mon[41183]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:40.991 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:40 smithi181 ceph-mon[41183]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:51:40.991 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:40 smithi181 ceph-mon[41183]: Reconfiguring daemon alertmanager.a on smithi181 2024-03-29T04:51:41.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:40 smithi187 ceph-mon[33595]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:41.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:40 smithi187 ceph-mon[33595]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:51:41.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:40 smithi187 ceph-mon[33595]: Reconfiguring daemon alertmanager.a on smithi181 2024-03-29T04:51:41.647 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:42.049 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:41 smithi181 ceph-mon[29765]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:51:42.049 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:41 smithi181 ceph-mon[41183]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:51:42.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:41 smithi187 ceph-mon[33595]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:51:43.086 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:42 smithi181 ceph-mon[41183]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:43.086 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:42 smithi181 ceph-mon[29765]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:43.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:42 smithi187 ceph-mon[33595]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:45.045 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:44 smithi181 ceph-mon[41183]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:45.045 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:44 smithi181 ceph-mon[29765]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:45.045 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:44 smithi181 systemd[1]: Stopping Ceph alertmanager.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:51:45.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:44 smithi187 ceph-mon[33595]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:45.538 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:51:45.538 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-29T04:51:45.910 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:45 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[72459]: ts=2024-03-29T04:51:45.452Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:51:45.910 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:45 smithi181 podman[77999]: 2024-03-29 04:51:45.483288553 +0000 UTC m=+0.177112904 container died 79e961bfe034831b50ec306b60700047b537dd433482668a330da0dc476b3f4c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:51:46.168 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:45 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/305986988' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:51:46.169 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:46 smithi181 podman[77999]: 2024-03-29 04:51:46.143098162 +0000 UTC m=+0.836922498 container cleanup 79e961bfe034831b50ec306b60700047b537dd433482668a330da0dc476b3f4c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:51:46.169 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:46 smithi181 bash[77999]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a 2024-03-29T04:51:46.169 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:45 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/305986988' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:51:46.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:45 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/305986988' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:51:46.599 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-03-29T04:51:46.599 INFO:tasks.cephadm:Setup complete, yielding 2024-03-29T04:51:46.600 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-29T04:51:46.609 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi181.front.sepia.ceph.com 2024-03-29T04:51:46.610 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- bash -c 'ceph fs volume create foo' 2024-03-29T04:51:46.876 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:46 smithi181 podman[78019]: 2024-03-29 04:51:46.788763456 +0000 UTC m=+1.309943702 container remove 79e961bfe034831b50ec306b60700047b537dd433482668a330da0dc476b3f4c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:51:46.876 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:46 smithi181 podman[78019]: 2024-03-29 04:51:46.872677758 +0000 UTC m=+1.393857994 volume remove afb1cd572f61bf3de99a440ccb68335689d22ec06e95d6c91bc92b9bd326019b 2024-03-29T04:51:46.963 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:47.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:46 smithi181 ceph-mon[29765]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:47.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:46 smithi181 ceph-mon[41183]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:47.200 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:46 smithi181 bash[78105]: Error: no container with name or ID "ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a" found: no such container 2024-03-29T04:51:47.200 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:46 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-03-29T04:51:47.200 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:46 smithi181 systemd[1]: Stopped Ceph alertmanager.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:51:47.200 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:46 smithi181 systemd[1]: Starting Ceph alertmanager.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:51:47.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:46 smithi187 ceph-mon[33595]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:47.623 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:47 smithi181 podman[78227]: 2024-03-29 04:51:47.212538069 +0000 UTC m=+0.019365332 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-03-29T04:51:47.623 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:47 smithi181 podman[78227]: 2024-03-29 04:51:47.325609288 +0000 UTC m=+0.132436550 volume create e8575d0e0bec37bd2f5e334c7deb5eae75090e23fe84ec8b06096f8f7699ae20 2024-03-29T04:51:47.623 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:47 smithi181 podman[78227]: 2024-03-29 04:51:47.426003914 +0000 UTC m=+0.232831222 container create d22dcca4e726cd0151679ffa1cdc5a9c41556c3cc5a36e66105f0d9c6c59d50e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:51:48.001 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:47 smithi181 podman[78227]: 2024-03-29 04:51:47.728765038 +0000 UTC m=+0.535592299 container init d22dcca4e726cd0151679ffa1cdc5a9c41556c3cc5a36e66105f0d9c6c59d50e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:51:48.001 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:47 smithi181 podman[78227]: 2024-03-29 04:51:47.734268548 +0000 UTC m=+0.541095812 container start d22dcca4e726cd0151679ffa1cdc5a9c41556c3cc5a36e66105f0d9c6c59d50e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:51:48.001 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:47 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[78260]: ts=2024-03-29T04:51:47.746Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-03-29T04:51:48.001 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:47 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[78260]: ts=2024-03-29T04:51:47.746Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-03-29T04:51:48.001 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:47 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[78260]: ts=2024-03-29T04:51:47.747Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.181 port=9094 2024-03-29T04:51:48.001 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:47 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[78260]: ts=2024-03-29T04:51:47.748Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-03-29T04:51:48.001 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:47 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[78260]: ts=2024-03-29T04:51:47.771Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:51:48.001 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:47 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[78260]: ts=2024-03-29T04:51:47.771Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:51:48.001 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:47 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[78260]: ts=2024-03-29T04:51:47.774Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-03-29T04:51:48.001 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:47 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[78260]: ts=2024-03-29T04:51:47.774Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-03-29T04:51:48.001 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:47 smithi181 bash[78227]: d22dcca4e726cd0151679ffa1cdc5a9c41556c3cc5a36e66105f0d9c6c59d50e 2024-03-29T04:51:48.001 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:47 smithi181 systemd[1]: Started Ceph alertmanager.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:51:48.798 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:51:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:51:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:51:49.099 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:48 smithi181 ceph-mon[29765]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:49.099 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:48 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:49.099 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:48 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:49.099 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:48 smithi181 ceph-mon[29765]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:51:49.099 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:48 smithi181 ceph-mon[29765]: Reconfiguring daemon prometheus.a on smithi187 2024-03-29T04:51:49.099 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:48 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:49.099 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:48 smithi181 ceph-mon[41183]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:49.099 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:48 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:49.099 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:48 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:49.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:48 smithi181 ceph-mon[41183]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:51:49.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:48 smithi181 ceph-mon[41183]: Reconfiguring daemon prometheus.a on smithi187 2024-03-29T04:51:49.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:48 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:49.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:48 smithi187 ceph-mon[33595]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:49.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:48 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:49.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:48 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:49.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:48 smithi187 ceph-mon[33595]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:51:49.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:48 smithi187 ceph-mon[33595]: Reconfiguring daemon prometheus.a on smithi187 2024-03-29T04:51:49.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:48 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:50.129 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:49 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-29T04:51:50.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:49 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-29T04:51:50.200 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:49 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[78260]: ts=2024-03-29T04:51:49.749Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000847548s 2024-03-29T04:51:50.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:49 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-29T04:51:50.977 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:50 smithi187 ceph-mon[33595]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:51:50.977 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:50 smithi187 ceph-mon[33595]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:50.977 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:50 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-03-29T04:51:50.977 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:50 smithi187 ceph-mon[33595]: osdmap e65: 8 total, 8 up, 8 in 2024-03-29T04:51:50.977 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:50 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-29T04:51:51.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:50 smithi181 ceph-mon[41183]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:51:51.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:50 smithi181 ceph-mon[41183]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:51.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:50 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-03-29T04:51:51.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:50 smithi181 ceph-mon[41183]: osdmap e65: 8 total, 8 up, 8 in 2024-03-29T04:51:51.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:50 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-29T04:51:51.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:50 smithi181 ceph-mon[29765]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:51:51.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:50 smithi181 ceph-mon[29765]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:51.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:50 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-03-29T04:51:51.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:50 smithi181 ceph-mon[29765]: osdmap e65: 8 total, 8 up, 8 in 2024-03-29T04:51:51.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:50 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-29T04:51:51.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:50 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a[29742]: 2024-03-29T04:51:50.886+0000 7fe46468e700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-29T04:51:51.380 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:51 smithi187 systemd[1]: Stopping Ceph prometheus.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:51:51.680 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-03-29T04:51:51.880 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:51:51.613Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:51:51.880 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:51:51.614Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-29T04:51:51.880 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:51:51.614Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-29T04:51:51.880 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:51:51.614Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-29T04:51:51.881 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:51:51.614Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-29T04:51:51.881 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:51:51.614Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-29T04:51:51.881 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:51:51.614Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-29T04:51:51.881 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:51:51.614Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-29T04:51:51.881 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:51:51.614Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-29T04:51:51.881 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:51:51.614Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-29T04:51:51.881 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:51:51.614Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-29T04:51:51.881 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[63485]: ts=2024-03-29T04:51:51.614Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-29T04:51:51.881 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:51 smithi187 podman[66937]: 2024-03-29 04:51:51.624432163 +0000 UTC m=+0.372393345 container died 1be4c994efc472443d58ab87580057935c07836ad701ba4d1ebad66c2a0c3647 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:51:51.943 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:51.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-03-29T04:51:51.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[29765]: osdmap e66: 8 total, 8 up, 8 in 2024-03-29T04:51:51.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-29T04:51:51.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[29765]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-29T04:51:51.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[29765]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-29T04:51:51.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[29765]: osdmap e67: 8 total, 8 up, 8 in 2024-03-29T04:51:51.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-03-29T04:51:51.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[29765]: fsmap foo:0 2024-03-29T04:51:51.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[29765]: Saving service mds.foo spec with placement count:2 2024-03-29T04:51:51.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:51.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-03-29T04:51:51.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[41183]: osdmap e66: 8 total, 8 up, 8 in 2024-03-29T04:51:51.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-29T04:51:51.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[41183]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-29T04:51:51.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[41183]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-29T04:51:51.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[41183]: osdmap e67: 8 total, 8 up, 8 in 2024-03-29T04:51:51.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-03-29T04:51:51.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[41183]: fsmap foo:0 2024-03-29T04:51:51.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[41183]: Saving service mds.foo spec with placement count:2 2024-03-29T04:51:51.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:51 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:52.274 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-03-29T04:51:52.274 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-mon[33595]: osdmap e66: 8 total, 8 up, 8 in 2024-03-29T04:51:52.274 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-29T04:51:52.274 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-mon[33595]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-29T04:51:52.274 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-mon[33595]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-29T04:51:52.275 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-mon[33595]: osdmap e67: 8 total, 8 up, 8 in 2024-03-29T04:51:52.275 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-03-29T04:51:52.275 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-mon[33595]: fsmap foo:0 2024-03-29T04:51:52.275 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-mon[33595]: Saving service mds.foo spec with placement count:2 2024-03-29T04:51:52.275 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:51 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:52.275 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:51 smithi187 podman[66937]: 2024-03-29 04:51:51.994310415 +0000 UTC m=+0.742271587 container cleanup 1be4c994efc472443d58ab87580057935c07836ad701ba4d1ebad66c2a0c3647 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:51:52.275 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:51 smithi187 bash[66937]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a 2024-03-29T04:51:52.625 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 podman[66956]: 2024-03-29 04:51:52.270662423 +0000 UTC m=+0.645473160 container remove 1be4c994efc472443d58ab87580057935c07836ad701ba4d1ebad66c2a0c3647 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:51:52.625 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-03-29T04:51:52.625 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 systemd[1]: Stopped Ceph prometheus.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:51:52.625 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@prometheus.a.service: Consumed 1.101s CPU time. 2024-03-29T04:51:52.625 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 systemd[1]: Starting Ceph prometheus.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:51:52.877 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 podman[67050]: 2024-03-29 04:51:52.524774699 +0000 UTC m=+0.018251684 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-03-29T04:51:52.877 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 podman[67050]: 2024-03-29 04:51:52.646779235 +0000 UTC m=+0.140256266 container create f7620d3fb68f42e2e270de1a462c2e3a2cf697ea53ddc39a690904bedd1e5546 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:51:53.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-mon[33595]: pgmap v49: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:53.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-mon[33595]: osdmap e68: 8 total, 8 up, 8 in 2024-03-29T04:51:53.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 podman[67050]: 2024-03-29 04:51:52.874008463 +0000 UTC m=+0.367485461 container init f7620d3fb68f42e2e270de1a462c2e3a2cf697ea53ddc39a690904bedd1e5546 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:51:53.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 podman[67050]: 2024-03-29 04:51:52.87968201 +0000 UTC m=+0.373158998 container start f7620d3fb68f42e2e270de1a462c2e3a2cf697ea53ddc39a690904bedd1e5546 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:51:53.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.904Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-29T04:51:53.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.904Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-29T04:51:53.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.904Z caller=main.go:570 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi187 (none))" 2024-03-29T04:51:53.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.904Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-29T04:51:53.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.904Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-29T04:51:53.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.905Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-29T04:51:53.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.905Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-29T04:51:53.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.906Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-29T04:51:53.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.906Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-29T04:51:53.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.908Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-29T04:51:53.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.908Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.808µs 2024-03-29T04:51:53.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.908Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-29T04:51:53.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.909Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-03-29T04:51:53.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.909Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-03-29T04:51:53.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.909Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.641µs wal_replay_duration=522.038µs wbl_replay_duration=101ns total_replay_duration=563.23µs 2024-03-29T04:51:53.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.910Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-29T04:51:53.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.910Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-29T04:51:53.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.910Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-29T04:51:53.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.925Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.579864ms db_storage=747ns remote_storage=1.328µs web_handler=348ns query_engine=669ns scrape=400.122µs scrape_sd=34.203µs notify=18.812µs notify_sd=5.555µs rules=14.907896ms tracing=5.634µs 2024-03-29T04:51:53.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.925Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-29T04:51:53.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:51:52.925Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-29T04:51:53.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:53 smithi187 bash[67050]: f7620d3fb68f42e2e270de1a462c2e3a2cf697ea53ddc39a690904bedd1e5546 2024-03-29T04:51:53.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:51:53 smithi187 systemd[1]: Started Ceph prometheus.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:51:53.198 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:52 smithi181 ceph-mon[29765]: pgmap v49: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:53.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:52 smithi181 ceph-mon[29765]: osdmap e68: 8 total, 8 up, 8 in 2024-03-29T04:51:53.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:52 smithi181 ceph-mon[41183]: pgmap v49: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:53.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:52 smithi181 ceph-mon[41183]: osdmap e68: 8 total, 8 up, 8 in 2024-03-29T04:51:53.199 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE Bus STOPPING 2024-03-29T04:51:53.199 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:51:53.199 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE Bus STOPPED 2024-03-29T04:51:53.199 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE Bus STARTING 2024-03-29T04:51:53.200 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE Serving on http://:::9283 2024-03-29T04:51:53.450 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE Bus STARTED 2024-03-29T04:51:53.450 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE Bus STOPPING 2024-03-29T04:51:53.450 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:51:53.450 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE Bus STOPPED 2024-03-29T04:51:53.450 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE Bus STARTING 2024-03-29T04:51:53.450 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE Serving on http://:::9283 2024-03-29T04:51:53.450 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE Bus STARTED 2024-03-29T04:51:53.450 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE Bus STOPPING 2024-03-29T04:51:53.451 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:51:53.451 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE Bus STOPPED 2024-03-29T04:51:53.451 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE Bus STARTING 2024-03-29T04:51:53.451 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE Serving on http://:::9283 2024-03-29T04:51:53.451 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:51:53] ENGINE Bus STARTED 2024-03-29T04:51:54.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:53 smithi187 ceph-mon[33595]: osdmap e69: 8 total, 8 up, 8 in 2024-03-29T04:51:54.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:53 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:54.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:53 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:54.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:53 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:51:54.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:53 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi181.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:51:54.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:53 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:54.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:53 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:51:54.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:53 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi187.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:51:54.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:53 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:54.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:53 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:51:54.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:53 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi187.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:51:54.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:53 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:54.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:53 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:54.171 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[41183]: osdmap e69: 8 total, 8 up, 8 in 2024-03-29T04:51:54.171 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:54.171 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:54.171 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:51:54.172 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi181.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:51:54.172 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:54.172 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:51:54.172 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi187.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:51:54.172 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:54.172 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:51:54.172 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi187.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:51:54.172 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:54.172 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:54.172 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[29765]: osdmap e69: 8 total, 8 up, 8 in 2024-03-29T04:51:54.172 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:54.173 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:54.173 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:51:54.173 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi181.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:51:54.173 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:54.173 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:51:54.173 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi187.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:51:54.173 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:54.173 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:51:54.173 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi187.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:51:54.173 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:54.173 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:53 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:54.783 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-03-29T04:51:55.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:54 smithi187 ceph-mon[33595]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:51:55.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:54 smithi187 ceph-mon[33595]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi181.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:51:55.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:54 smithi187 ceph-mon[33595]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:51:55.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:54 smithi187 ceph-mon[33595]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi187.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:51:55.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:54 smithi187 ceph-mon[33595]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:51:55.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:54 smithi187 ceph-mon[33595]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi187.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:51:55.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:54 smithi187 ceph-mon[33595]: pgmap v52: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:55.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:54 smithi181 ceph-mon[41183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:51:55.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:54 smithi181 ceph-mon[41183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi181.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:51:55.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:54 smithi181 ceph-mon[41183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:51:55.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:54 smithi181 ceph-mon[41183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi187.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:51:55.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:54 smithi181 ceph-mon[41183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:51:55.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:54 smithi181 ceph-mon[41183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi187.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:51:55.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:54 smithi181 ceph-mon[41183]: pgmap v52: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:55.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:54 smithi181 ceph-mon[29765]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:51:55.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:54 smithi181 ceph-mon[29765]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi181.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:51:55.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:54 smithi181 ceph-mon[29765]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:51:55.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:54 smithi181 ceph-mon[29765]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi187.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:51:55.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:54 smithi181 ceph-mon[29765]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:51:55.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:54 smithi181 ceph-mon[29765]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi187.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:51:55.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:54 smithi181 ceph-mon[29765]: pgmap v52: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:55.278 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:57.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:56 smithi181 ceph-mon[29765]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:57.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:56 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:57.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:56 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:57.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:56 smithi181 ceph-mon[41183]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:57.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:56 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:57.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:56 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:57.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:56 smithi187 ceph-mon[33595]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:57.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:56 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:57.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:56 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:51:57.950 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:51:57 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[78260]: ts=2024-03-29T04:51:57.752Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003980806s 2024-03-29T04:51:58.371 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-03-29T04:51:58.382 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-29T04:51:58.382 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-29T04:51:58.382 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph mgr dump --format=json 2024-03-29T04:51:58.880 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:51:58 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:51:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:51:59.063 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:59.156 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:51:58 smithi187 ceph-mon[33595]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:59.186 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:51:58 smithi181 ceph-mon[41183]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:51:59.187 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:51:58 smithi181 ceph-mon[29765]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:00.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:00.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:00.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:00.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:00.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:00.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.pflrwm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:52:00.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.pflrwm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:52:00.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:00.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[29765]: Deploying daemon mds.foo.smithi181.pflrwm on smithi181 2024-03-29T04:52:00.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[29765]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:00.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:00.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:00.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:00.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:00.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:00.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.pflrwm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:52:00.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.pflrwm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:52:00.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:00.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[41183]: Deploying daemon mds.foo.smithi181.pflrwm on smithi181 2024-03-29T04:52:00.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:00 smithi181 ceph-mon[41183]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:00.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:00 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:00.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:00 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:00.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:00 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:00.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:00 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:00.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:00 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:00.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:00 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.pflrwm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:52:00.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:00 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.pflrwm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:52:00.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:00 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:00.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:00 smithi187 ceph-mon[33595]: Deploying daemon mds.foo.smithi181.pflrwm on smithi181 2024-03-29T04:52:00.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:00 smithi187 ceph-mon[33595]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:01.098 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:52:01.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:01 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1233132361' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:52:01.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:01 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1233132361' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:52:01.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:01 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1233132361' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:52:02.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:02 smithi181 ceph-mon[29765]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:02.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:02 smithi181 ceph-mon[41183]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:02.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:02 smithi187 ceph-mon[33595]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:02.713 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":20,"active_gid":14448,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":3281522413},{"type":"v1","addr":"172.21.15.181:6801","nonce":3281522413}]},"active_addr":"172.21.15.181:6801/3281522413","active_change":"2024-03-29T04:50:33.576252+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24334,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.181:8443/","prometheus":"http://172.21.15.181:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.181:0","nonce":3860889501}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.181:0","nonce":1741596559}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.181:0","nonce":2280393448}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.181:0","nonce":1220271702}]}]} 2024-03-29T04:52:02.718 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-29T04:52:02.718 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-29T04:52:02.719 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:52:03.294 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:04.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: mds.? [v2:172.21.15.181:6834/2778467308,v1:172.21.15.181:6835/2778467308] up:boot 2024-03-29T04:52:04.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: daemon mds.foo.smithi181.pflrwm assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-03-29T04:52:04.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-29T04:52:04.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-29T04:52:04.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: Cluster is now healthy 2024-03-29T04:52:04.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: fsmap foo:0 1 up:standby 2024-03-29T04:52:04.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.pflrwm"}]: dispatch 2024-03-29T04:52:04.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:creating} 2024-03-29T04:52:04.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: daemon mds.foo.smithi181.pflrwm is now active in filesystem foo as rank 0 2024-03-29T04:52:04.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:04.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:04.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:04.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi187.vcsnqx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:52:04.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi187.vcsnqx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:52:04.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:04.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: Deploying daemon mds.foo.smithi187.vcsnqx on smithi187 2024-03-29T04:52:04.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:04.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:04.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: mds.? [v2:172.21.15.181:6834/2778467308,v1:172.21.15.181:6835/2778467308] up:boot 2024-03-29T04:52:04.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: daemon mds.foo.smithi181.pflrwm assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-03-29T04:52:04.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-29T04:52:04.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-29T04:52:04.450 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: Cluster is now healthy 2024-03-29T04:52:04.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: fsmap foo:0 1 up:standby 2024-03-29T04:52:04.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.pflrwm"}]: dispatch 2024-03-29T04:52:04.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:creating} 2024-03-29T04:52:04.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: daemon mds.foo.smithi181.pflrwm is now active in filesystem foo as rank 0 2024-03-29T04:52:04.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:04.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:04.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:04.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi187.vcsnqx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:52:04.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi187.vcsnqx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:52:04.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:04.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: Deploying daemon mds.foo.smithi187.vcsnqx on smithi187 2024-03-29T04:52:04.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:04.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:04.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: mds.? [v2:172.21.15.181:6834/2778467308,v1:172.21.15.181:6835/2778467308] up:boot 2024-03-29T04:52:04.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: daemon mds.foo.smithi181.pflrwm assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-03-29T04:52:04.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-29T04:52:04.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-29T04:52:04.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: Cluster is now healthy 2024-03-29T04:52:04.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: fsmap foo:0 1 up:standby 2024-03-29T04:52:04.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.pflrwm"}]: dispatch 2024-03-29T04:52:04.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:creating} 2024-03-29T04:52:04.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: daemon mds.foo.smithi181.pflrwm is now active in filesystem foo as rank 0 2024-03-29T04:52:04.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:04.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:04.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:04.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi187.vcsnqx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:52:04.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi187.vcsnqx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:52:04.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:04.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: Deploying daemon mds.foo.smithi187.vcsnqx on smithi187 2024-03-29T04:52:04.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:04.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:04.852 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:52:04 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:52:04] "GET /metrics HTTP/1.1" 200 208342 "" "Prometheus/2.43.0" 2024-03-29T04:52:05.302 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:52:05.302 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":69,"fsid":"dfcecab8-ed86-11ee-95d0-87774f69a715","created":"2024-03-29T04:44:49.014354+0000","modified":"2024-03-29T04:51:52.906488+0000","last_up_change":"2024-03-29T04:50:02.141186+0000","last_in_change":"2024-03-29T04:49:44.602034+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:48:01.551395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:50:37.400395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-29T04:51:49.771458+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-29T04:51:50.114853+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"26991441-a6cb-4694-aec3-0f793de6d551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6803","nonce":135202285}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6805","nonce":135202285}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6809","nonce":135202285}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6807","nonce":135202285}]},"public_addr":"172.21.15.181:6803/135202285","cluster_addr":"172.21.15.181:6805/135202285","heartbeat_back_addr":"172.21.15.181:6809/135202285","heartbeat_front_addr":"172.21.15.181:6807/135202285","state":["exists","up"]},{"osd":1,"uuid":"04b645ce-4eb8-43bc-9f28-e3be932f4b1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6811","nonce":3021678652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6813","nonce":3021678652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6817","nonce":3021678652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6815","nonce":3021678652}]},"public_addr":"172.21.15.181:6811/3021678652","cluster_addr":"172.21.15.181:6813/3021678652","heartbeat_back_addr":"172.21.15.181:6817/3021678652","heartbeat_front_addr":"172.21.15.181:6815/3021678652","state":["exists","up"]},{"osd":2,"uuid":"f72d641c-5d86-4d3a-bf90-d99c1c46a805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6819","nonce":352673207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6821","nonce":352673207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6825","nonce":352673207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6823","nonce":352673207}]},"public_addr":"172.21.15.181:6819/352673207","cluster_addr":"172.21.15.181:6821/352673207","heartbeat_back_addr":"172.21.15.181:6825/352673207","heartbeat_front_addr":"172.21.15.181:6823/352673207","state":["exists","up"]},{"osd":3,"uuid":"63ebbc1a-43b5-4990-87aa-f8f0ff975830","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6827","nonce":1785806254}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6829","nonce":1785806254}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6832","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6833","nonce":1785806254}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6831","nonce":1785806254}]},"public_addr":"172.21.15.181:6827/1785806254","cluster_addr":"172.21.15.181:6829/1785806254","heartbeat_back_addr":"172.21.15.181:6833/1785806254","heartbeat_front_addr":"172.21.15.181:6831/1785806254","state":["exists","up"]},{"osd":4,"uuid":"0a71f228-2cc8-4b83-b86d-62a2854a8533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6800","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6801","nonce":3700510703}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6802","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6803","nonce":3700510703}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6806","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6807","nonce":3700510703}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6804","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6805","nonce":3700510703}]},"public_addr":"172.21.15.187:6801/3700510703","cluster_addr":"172.21.15.187:6803/3700510703","heartbeat_back_addr":"172.21.15.187:6807/3700510703","heartbeat_front_addr":"172.21.15.187:6805/3700510703","state":["exists","up"]},{"osd":5,"uuid":"847657e4-7b9a-4eee-9df5-8e32d0d6543e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6808","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6809","nonce":989620923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6810","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6811","nonce":989620923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6814","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6815","nonce":989620923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6812","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6813","nonce":989620923}]},"public_addr":"172.21.15.187:6809/989620923","cluster_addr":"172.21.15.187:6811/989620923","heartbeat_back_addr":"172.21.15.187:6815/989620923","heartbeat_front_addr":"172.21.15.187:6813/989620923","state":["exists","up"]},{"osd":6,"uuid":"cead714e-ffc1-44a4-bef5-cfa772070c9f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6816","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6817","nonce":1708678714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6818","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6819","nonce":1708678714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6822","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6823","nonce":1708678714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6820","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6821","nonce":1708678714}]},"public_addr":"172.21.15.187:6817/1708678714","cluster_addr":"172.21.15.187:6819/1708678714","heartbeat_back_addr":"172.21.15.187:6823/1708678714","heartbeat_front_addr":"172.21.15.187:6821/1708678714","state":["exists","up"]},{"osd":7,"uuid":"9e2520cb-e4ba-4b2a-9206-8a250e7a373f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6824","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6825","nonce":2139604103}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6826","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6827","nonce":2139604103}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6830","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6831","nonce":2139604103}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6828","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6829","nonce":2139604103}]},"public_addr":"172.21.15.187:6825/2139604103","cluster_addr":"172.21.15.187:6827/2139604103","heartbeat_back_addr":"172.21.15.187:6831/2139604103","heartbeat_front_addr":"172.21.15.187:6829/2139604103","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:08.364426+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:33.120007+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:57.511900+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:48:22.235902+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:48:45.159568+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:49:10.475329+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:49:34.890974+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:50:00.243723+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.181:6800/3044637144":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/4244016576":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/3491810134":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/3153434425":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/114113941":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/2540916225":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/2863027222":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/19173868":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/1768629494":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6800/2863027222":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/2056930738":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/4052605721":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/1867140711":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6800/2540916225":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/3044637144":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/196688700":"2024-03-30T04:45:25.467994+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:52:05.501 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:05 smithi187 ceph-mon[33595]: mds.? [v2:172.21.15.181:6834/2778467308,v1:172.21.15.181:6835/2778467308] up:active 2024-03-29T04:52:05.501 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:05 smithi187 ceph-mon[33595]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 2024-03-29T04:52:05.675 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:05 smithi181 ceph-mon[41183]: mds.? [v2:172.21.15.181:6834/2778467308,v1:172.21.15.181:6835/2778467308] up:active 2024-03-29T04:52:05.676 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:05 smithi181 ceph-mon[41183]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 2024-03-29T04:52:05.676 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:05 smithi181 ceph-mon[29765]: mds.? [v2:172.21.15.181:6834/2778467308,v1:172.21.15.181:6835/2778467308] up:active 2024-03-29T04:52:05.676 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:05 smithi181 ceph-mon[29765]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 2024-03-29T04:52:06.094 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-29T04:52:06.094 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:52:06.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:06 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/642237813' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:52:06.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:06 smithi181 ceph-mon[29765]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:06.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:06 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:06.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:06 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:06.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:06 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:06.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:06 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:06.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:06 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:06.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:06 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:06.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:06 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/642237813' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:52:06.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:06 smithi181 ceph-mon[41183]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:06.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:06 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:06.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:06 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:06.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:06 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:06.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:06 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:06.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:06 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:06.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:06 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:06.533 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:06 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/642237813' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:52:06.533 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:06 smithi187 ceph-mon[33595]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:06.533 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:06 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:06.533 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:06 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:06.533 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:06 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:06.533 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:06 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:06.533 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:06 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:06.534 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:06 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:06.975 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:07.384 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:07 smithi181 ceph-mon[29765]: mds.? [v2:172.21.15.187:6832/1383464008,v1:172.21.15.187:6833/1383464008] up:boot 2024-03-29T04:52:07.384 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:07 smithi181 ceph-mon[29765]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:52:07.384 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:07 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi187.vcsnqx"}]: dispatch 2024-03-29T04:52:07.384 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:07 smithi181 ceph-mon[29765]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:52:07.384 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:07 smithi181 ceph-mon[41183]: mds.? [v2:172.21.15.187:6832/1383464008,v1:172.21.15.187:6833/1383464008] up:boot 2024-03-29T04:52:07.384 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:07 smithi181 ceph-mon[41183]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:52:07.384 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:07 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi187.vcsnqx"}]: dispatch 2024-03-29T04:52:07.385 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:07 smithi181 ceph-mon[41183]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:52:07.521 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:07 smithi187 ceph-mon[33595]: mds.? [v2:172.21.15.187:6832/1383464008,v1:172.21.15.187:6833/1383464008] up:boot 2024-03-29T04:52:07.521 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:07 smithi187 ceph-mon[33595]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:52:07.521 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:07 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi187.vcsnqx"}]: dispatch 2024-03-29T04:52:07.521 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:07 smithi187 ceph-mon[33595]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:52:08.537 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:08 smithi187 ceph-mon[33595]: mds.? [v2:172.21.15.181:6834/2778467308,v1:172.21.15.181:6835/2778467308] up:active 2024-03-29T04:52:08.537 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:08 smithi187 ceph-mon[33595]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:52:08.537 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:08 smithi187 ceph-mon[33595]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:08.541 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:08 smithi181 ceph-mon[41183]: mds.? [v2:172.21.15.181:6834/2778467308,v1:172.21.15.181:6835/2778467308] up:active 2024-03-29T04:52:08.541 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:08 smithi181 ceph-mon[41183]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:52:08.541 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:08 smithi181 ceph-mon[41183]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:08.541 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:08 smithi181 ceph-mon[29765]: mds.? [v2:172.21.15.181:6834/2778467308,v1:172.21.15.181:6835/2778467308] up:active 2024-03-29T04:52:08.542 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:08 smithi181 ceph-mon[29765]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:52:08.542 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:08 smithi181 ceph-mon[29765]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:08.880 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:52:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:52:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:52:09.143 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:52:09.144 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":69,"fsid":"dfcecab8-ed86-11ee-95d0-87774f69a715","created":"2024-03-29T04:44:49.014354+0000","modified":"2024-03-29T04:51:52.906488+0000","last_up_change":"2024-03-29T04:50:02.141186+0000","last_in_change":"2024-03-29T04:49:44.602034+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:48:01.551395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:50:37.400395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-29T04:51:49.771458+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-29T04:51:50.114853+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"26991441-a6cb-4694-aec3-0f793de6d551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6803","nonce":135202285}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6805","nonce":135202285}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6809","nonce":135202285}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":135202285},{"type":"v1","addr":"172.21.15.181:6807","nonce":135202285}]},"public_addr":"172.21.15.181:6803/135202285","cluster_addr":"172.21.15.181:6805/135202285","heartbeat_back_addr":"172.21.15.181:6809/135202285","heartbeat_front_addr":"172.21.15.181:6807/135202285","state":["exists","up"]},{"osd":1,"uuid":"04b645ce-4eb8-43bc-9f28-e3be932f4b1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6811","nonce":3021678652}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6813","nonce":3021678652}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6817","nonce":3021678652}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":3021678652},{"type":"v1","addr":"172.21.15.181:6815","nonce":3021678652}]},"public_addr":"172.21.15.181:6811/3021678652","cluster_addr":"172.21.15.181:6813/3021678652","heartbeat_back_addr":"172.21.15.181:6817/3021678652","heartbeat_front_addr":"172.21.15.181:6815/3021678652","state":["exists","up"]},{"osd":2,"uuid":"f72d641c-5d86-4d3a-bf90-d99c1c46a805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6819","nonce":352673207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6821","nonce":352673207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6825","nonce":352673207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":352673207},{"type":"v1","addr":"172.21.15.181:6823","nonce":352673207}]},"public_addr":"172.21.15.181:6819/352673207","cluster_addr":"172.21.15.181:6821/352673207","heartbeat_back_addr":"172.21.15.181:6825/352673207","heartbeat_front_addr":"172.21.15.181:6823/352673207","state":["exists","up"]},{"osd":3,"uuid":"63ebbc1a-43b5-4990-87aa-f8f0ff975830","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6827","nonce":1785806254}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6829","nonce":1785806254}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6832","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6833","nonce":1785806254}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":1785806254},{"type":"v1","addr":"172.21.15.181:6831","nonce":1785806254}]},"public_addr":"172.21.15.181:6827/1785806254","cluster_addr":"172.21.15.181:6829/1785806254","heartbeat_back_addr":"172.21.15.181:6833/1785806254","heartbeat_front_addr":"172.21.15.181:6831/1785806254","state":["exists","up"]},{"osd":4,"uuid":"0a71f228-2cc8-4b83-b86d-62a2854a8533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6800","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6801","nonce":3700510703}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6802","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6803","nonce":3700510703}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6806","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6807","nonce":3700510703}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6804","nonce":3700510703},{"type":"v1","addr":"172.21.15.187:6805","nonce":3700510703}]},"public_addr":"172.21.15.187:6801/3700510703","cluster_addr":"172.21.15.187:6803/3700510703","heartbeat_back_addr":"172.21.15.187:6807/3700510703","heartbeat_front_addr":"172.21.15.187:6805/3700510703","state":["exists","up"]},{"osd":5,"uuid":"847657e4-7b9a-4eee-9df5-8e32d0d6543e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6808","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6809","nonce":989620923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6810","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6811","nonce":989620923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6814","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6815","nonce":989620923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6812","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6813","nonce":989620923}]},"public_addr":"172.21.15.187:6809/989620923","cluster_addr":"172.21.15.187:6811/989620923","heartbeat_back_addr":"172.21.15.187:6815/989620923","heartbeat_front_addr":"172.21.15.187:6813/989620923","state":["exists","up"]},{"osd":6,"uuid":"cead714e-ffc1-44a4-bef5-cfa772070c9f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6816","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6817","nonce":1708678714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6818","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6819","nonce":1708678714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6822","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6823","nonce":1708678714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6820","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6821","nonce":1708678714}]},"public_addr":"172.21.15.187:6817/1708678714","cluster_addr":"172.21.15.187:6819/1708678714","heartbeat_back_addr":"172.21.15.187:6823/1708678714","heartbeat_front_addr":"172.21.15.187:6821/1708678714","state":["exists","up"]},{"osd":7,"uuid":"9e2520cb-e4ba-4b2a-9206-8a250e7a373f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6824","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6825","nonce":2139604103}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6826","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6827","nonce":2139604103}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6830","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6831","nonce":2139604103}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6828","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6829","nonce":2139604103}]},"public_addr":"172.21.15.187:6825/2139604103","cluster_addr":"172.21.15.187:6827/2139604103","heartbeat_back_addr":"172.21.15.187:6831/2139604103","heartbeat_front_addr":"172.21.15.187:6829/2139604103","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:08.364426+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:33.120007+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:57.511900+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:48:22.235902+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:48:45.159568+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:49:10.475329+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:49:34.890974+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:50:00.243723+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.181:6800/3044637144":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/4244016576":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/3491810134":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/3153434425":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/114113941":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/2540916225":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/2863027222":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/19173868":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/1768629494":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6800/2863027222":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/2056930738":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/4052605721":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/1867140711":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6800/2540916225":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/3044637144":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/196688700":"2024-03-30T04:45:25.467994+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:52:09.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:09 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:09.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:09 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1119491356' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:52:09.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:09 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:09.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:09 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:09.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:09 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:09.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:09 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1119491356' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:52:09.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:09 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:09.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:09 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:09.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:09 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:09.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:09 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1119491356' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:52:09.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:09 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:09.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:09 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:10.167 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-29T04:52:10.168 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-29T04:52:10.169 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-29T04:52:10.169 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-29T04:52:10.170 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-29T04:52:10.170 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-29T04:52:10.171 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-29T04:52:10.171 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-29T04:52:10.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:10 smithi181 ceph-mon[29765]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:10.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:10 smithi181 ceph-mon[29765]: mds.? [v2:172.21.15.187:6832/1383464008,v1:172.21.15.187:6833/1383464008] up:standby 2024-03-29T04:52:10.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:10 smithi181 ceph-mon[29765]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:52:10.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:10 smithi181 ceph-mon[41183]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:10.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:10 smithi181 ceph-mon[41183]: mds.? [v2:172.21.15.187:6832/1383464008,v1:172.21.15.187:6833/1383464008] up:standby 2024-03-29T04:52:10.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:10 smithi181 ceph-mon[41183]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:52:11.041 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:11.042 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:11.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:10 smithi187 ceph-mon[33595]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:11.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:10 smithi187 ceph-mon[33595]: mds.? [v2:172.21.15.187:6832/1383464008,v1:172.21.15.187:6833/1383464008] up:standby 2024-03-29T04:52:11.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:10 smithi187 ceph-mon[33595]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:52:11.321 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:12.595 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:12.595 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:12.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:12 smithi187 ceph-mon[33595]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:12.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:12 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:12.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:12 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:12.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:12 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:12.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:12 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:12.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:12 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:12.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:12 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:13.099 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:12 smithi181 ceph-mon[41183]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:13.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:12 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:13.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:12 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:13.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:12 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:13.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:12 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:13.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:12 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:13.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:12 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:13.100 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:12 smithi181 ceph-mon[29765]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:13.100 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:12 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:13.100 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:12 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:13.100 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:12 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:13.101 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:12 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:13.101 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:12 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:13.101 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:12 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:13.149 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:14.860 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:14 smithi181 ceph-mon[29765]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:14.860 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:52:14 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:52:14] "GET /metrics HTTP/1.1" 200 208342 "" "Prometheus/2.43.0" 2024-03-29T04:52:14.861 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:14 smithi181 ceph-mon[41183]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:15.052 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:14 smithi187 ceph-mon[33595]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:16.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:16 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:16.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:16 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:16.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:16 smithi181 ceph-mon[41183]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:16.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:16 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:16.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:16 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:16.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:16 smithi181 ceph-mon[29765]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:16.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:16 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:16.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:16 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:16.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:16 smithi187 ceph-mon[33595]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:52:17.221 INFO:teuthology.orchestra.run.smithi181.stdout:77309411383 2024-03-29T04:52:17.221 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-29T04:52:18.268 INFO:teuthology.orchestra.run.smithi181.stdout:55834574907 2024-03-29T04:52:18.268 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-29T04:52:18.284 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:18.662 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:18 smithi181 ceph-mon[29765]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:52:18.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:18 smithi187 ceph-mon[33595]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:52:18.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:18 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:18.880 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:52:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:52:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:52:18.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:18 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:18.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:18 smithi181 ceph-mon[41183]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:52:18.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:18 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:20.881 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:20 smithi181 ceph-mon[29765]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:52:20.881 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:20 smithi181 ceph-mon[41183]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:52:21.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:20 smithi187 ceph-mon[33595]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:52:22.043 INFO:teuthology.orchestra.run.smithi181.stdout:201863462948 2024-03-29T04:52:22.044 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-29T04:52:22.927 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:22 smithi181 ceph-mon[29765]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:22.928 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:22 smithi181 ceph-mon[41183]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:23.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:22 smithi187 ceph-mon[33595]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:23.336 INFO:teuthology.orchestra.run.smithi181.stdout:111669149747 2024-03-29T04:52:23.337 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-29T04:52:23.697 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:24.908 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:24 smithi181 ceph-mon[29765]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:24.909 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:52:24 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:52:24] "GET /metrics HTTP/1.1" 200 239117 "" "Prometheus/2.43.0" 2024-03-29T04:52:24.909 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:24 smithi181 ceph-mon[41183]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:25.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:24 smithi187 ceph-mon[33595]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:25.502 INFO:teuthology.orchestra.run.smithi181.stdout:34359738433 2024-03-29T04:52:25.503 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-29T04:52:26.942 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:26 smithi181 ceph-mon[41183]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:26.942 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:26 smithi181 ceph-mon[29765]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:27.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:26 smithi187 ceph-mon[33595]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:27.611 INFO:teuthology.orchestra.run.smithi181.stdout:167503724585 2024-03-29T04:52:27.612 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-29T04:52:28.458 INFO:teuthology.orchestra.run.smithi181.stdout:133143986222 2024-03-29T04:52:28.458 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-29T04:52:28.491 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:28.492 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:28.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:28 smithi181 ceph-mon[29765]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:28.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:28 smithi181 ceph-mon[41183]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:28.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:28 smithi187 ceph-mon[33595]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:28.880 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:52:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:52:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:52:28.905 INFO:teuthology.orchestra.run.smithi181.stdout:236223201311 2024-03-29T04:52:28.905 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-29T04:52:28.955 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:28.988 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:28.992 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:30.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:30 smithi181 ceph-mon[29765]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:30.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:30 smithi181 ceph-mon[41183]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:31.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:30 smithi187 ceph-mon[33595]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:31.975 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:31.978 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:32.700 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:32 smithi181 ceph-mon[29765]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:32.700 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:32 smithi181 ceph-mon[41183]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:33.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:32 smithi187 ceph-mon[33595]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:33.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:33 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-03-29T04:52:33.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:33 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-03-29T04:52:33.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:33 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-03-29T04:52:33.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:33 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:33.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:33 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-03-29T04:52:33.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:33 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-03-29T04:52:33.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:33 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-03-29T04:52:33.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:33 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:34.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:33 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-03-29T04:52:34.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:33 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-03-29T04:52:34.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:33 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-03-29T04:52:34.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:33 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:34.700 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:52:34 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:52:34] "GET /metrics HTTP/1.1" 200 239084 "" "Prometheus/2.43.0" 2024-03-29T04:52:34.900 INFO:teuthology.orchestra.run.smithi181.stdout:77309411386 2024-03-29T04:52:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:34 smithi181 ceph-mon[29765]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-03-29T04:52:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-03-29T04:52:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:34 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-03-29T04:52:34.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:34 smithi181 ceph-mon[29765]: osdmap e70: 8 total, 8 up, 8 in 2024-03-29T04:52:34.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:34 smithi181 ceph-mon[41183]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:34.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-03-29T04:52:34.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-03-29T04:52:34.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:34 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-03-29T04:52:34.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:34 smithi181 ceph-mon[41183]: osdmap e70: 8 total, 8 up, 8 in 2024-03-29T04:52:35.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:34 smithi187 ceph-mon[33595]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:35.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-03-29T04:52:35.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-03-29T04:52:35.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:34 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-03-29T04:52:35.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:34 smithi187 ceph-mon[33595]: osdmap e70: 8 total, 8 up, 8 in 2024-03-29T04:52:35.776 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:35 smithi181 ceph-mon[41183]: osdmap e71: 8 total, 8 up, 8 in 2024-03-29T04:52:35.776 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:35 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3209450944' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:52:35.776 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:35 smithi181 ceph-mon[29765]: osdmap e71: 8 total, 8 up, 8 in 2024-03-29T04:52:35.776 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:35 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3209450944' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:52:36.077 INFO:teuthology.orchestra.run.smithi181.stdout:55834574911 2024-03-29T04:52:36.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:35 smithi187 ceph-mon[33595]: osdmap e71: 8 total, 8 up, 8 in 2024-03-29T04:52:36.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:35 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3209450944' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:52:36.675 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411383 got 77309411386 for osd.2 2024-03-29T04:52:36.676 DEBUG:teuthology.parallel:result is None 2024-03-29T04:52:36.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:36 smithi181 ceph-mon[29765]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:36.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:36 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/4201068910' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:52:36.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:36 smithi181 ceph-mon[41183]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:36.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:36 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/4201068910' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:52:37.097 INFO:teuthology.orchestra.run.smithi181.stdout:201863462951 2024-03-29T04:52:37.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:36 smithi187 ceph-mon[33595]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:37.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:36 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/4201068910' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:52:37.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:37 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3325712794' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:52:37.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:37 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3325712794' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:52:38.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:37 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3325712794' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:52:38.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:38 smithi187 ceph-mon[33595]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:38.881 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:52:38 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:52:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:52:38.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:38 smithi181 ceph-mon[41183]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:38.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:38 smithi181 ceph-mon[29765]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:39.105 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462948 got 201863462951 for osd.6 2024-03-29T04:52:39.106 DEBUG:teuthology.parallel:result is None 2024-03-29T04:52:39.500 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574907 got 55834574911 for osd.1 2024-03-29T04:52:39.500 DEBUG:teuthology.parallel:result is None 2024-03-29T04:52:40.112 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:40.274 INFO:teuthology.orchestra.run.smithi181.stdout:34359738437 2024-03-29T04:52:40.925 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:40 smithi181 ceph-mon[29765]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:40.926 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:40 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3100102808' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:52:40.926 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:40 smithi181 ceph-mon[41183]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:40.926 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:40 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3100102808' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:52:40.953 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738433 got 34359738437 for osd.0 2024-03-29T04:52:40.954 DEBUG:teuthology.parallel:result is None 2024-03-29T04:52:41.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:40 smithi187 ceph-mon[33595]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:41.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:40 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3100102808' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:52:41.747 INFO:teuthology.orchestra.run.smithi181.stdout:133143986225 2024-03-29T04:52:42.350 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986222 got 133143986225 for osd.4 2024-03-29T04:52:42.350 DEBUG:teuthology.parallel:result is None 2024-03-29T04:52:42.986 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:42 smithi181 ceph-mon[41183]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:42.986 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:42 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1438714733' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:52:42.986 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:42 smithi181 ceph-mon[29765]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:42.987 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:42 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1438714733' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:52:43.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:42 smithi187 ceph-mon[33595]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:43.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:42 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1438714733' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:52:43.445 INFO:teuthology.orchestra.run.smithi181.stdout:111669149751 2024-03-29T04:52:43.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:43 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1393211253' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:52:43.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:43 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1393211253' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:52:44.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:43 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1393211253' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:52:44.386 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149747 got 111669149751 for osd.3 2024-03-29T04:52:44.386 DEBUG:teuthology.parallel:result is None 2024-03-29T04:52:44.700 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:52:44 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:52:44] "GET /metrics HTTP/1.1" 200 239084 "" "Prometheus/2.43.0" 2024-03-29T04:52:45.073 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:44 smithi181 ceph-mon[41183]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:45.073 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:44 smithi181 ceph-mon[29765]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:45.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:44 smithi187 ceph-mon[33595]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:45.694 INFO:teuthology.orchestra.run.smithi181.stdout:167503724588 2024-03-29T04:52:45.804 INFO:teuthology.orchestra.run.smithi181.stdout:236223201315 2024-03-29T04:52:45.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:45 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3153262023' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:52:45.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:45 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3153262023' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:52:46.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:45 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3153262023' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:52:46.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:46 smithi181 ceph-mon[41183]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:46.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:46 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3730972145' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:52:46.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:46 smithi181 ceph-mon[29765]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:46.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:46 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3730972145' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:52:47.030 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724585 got 167503724588 for osd.5 2024-03-29T04:52:47.030 DEBUG:teuthology.parallel:result is None 2024-03-29T04:52:47.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:46 smithi187 ceph-mon[33595]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:47.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:46 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3730972145' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:52:47.527 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201311 got 236223201315 for osd.7 2024-03-29T04:52:47.527 DEBUG:teuthology.parallel:result is None 2024-03-29T04:52:47.527 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-29T04:52:47.528 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T04:52:47.924 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:48.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:48 smithi187 ceph-mon[33595]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:48.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:48 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:48.881 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:52:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:52:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:52:49.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:48 smithi181 ceph-mon[29765]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:49.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:48 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:49.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:48 smithi181 ceph-mon[41183]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:49.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:48 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:50.038 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:49 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:50.038 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:49 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:50.038 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:49 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:50.038 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:49 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:50.039 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:49 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:50.039 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:49 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:50.039 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:49 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:50.039 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:49 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:50.039 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:49 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:50.039 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:49 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:50.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:49 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:50.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:49 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:50.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:49 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:50.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:49 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:50.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:49 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:52:50.503 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:52:50.504 INFO:teuthology.orchestra.run.smithi181.stderr:dumped all 2024-03-29T04:52:51.169 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:50 smithi181 ceph-mon[41183]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:51.170 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:50 smithi181 ceph-mon[29765]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:51.195 INFO:teuthology.orchestra.run.smithi181.stdout:{"pg_ready":true,"pg_map":{"version":82,"stamp":"2024-03-29T04:52:49.612451+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386320,"kb_used_data":7460,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574896,"statfs":{"total":767960285184,"available":765516693504,"internally_reserved":0,"allocated":7639040,"data_stored":4307046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002528"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689128+0000","last_change":"2024-03-29T04:51:50.905064+0000","last_active":"2024-03-29T04:52:33.689128+0000","last_peered":"2024-03-29T04:52:33.689128+0000","last_clean":"2024-03-29T04:52:33.689128+0000","last_became_active":"2024-03-29T04:51:50.904783+0000","last_became_peered":"2024-03-29T04:51:50.904783+0000","last_unstale":"2024-03-29T04:52:33.689128+0000","last_undegraded":"2024-03-29T04:52:33.689128+0000","last_fullsized":"2024-03-29T04:52:33.689128+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:15:06.303234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.691667+0000","last_change":"2024-03-29T04:51:50.887615+0000","last_active":"2024-03-29T04:52:33.691667+0000","last_peered":"2024-03-29T04:52:33.691667+0000","last_clean":"2024-03-29T04:52:33.691667+0000","last_became_active":"2024-03-29T04:51:50.887273+0000","last_became_peered":"2024-03-29T04:51:50.887273+0000","last_unstale":"2024-03-29T04:52:33.691667+0000","last_undegraded":"2024-03-29T04:52:33.691667+0000","last_fullsized":"2024-03-29T04:52:33.691667+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:24:36.868812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686457+0000","last_change":"2024-03-29T04:51:51.914368+0000","last_active":"2024-03-29T04:52:33.686457+0000","last_peered":"2024-03-29T04:52:33.686457+0000","last_clean":"2024-03-29T04:52:33.686457+0000","last_became_active":"2024-03-29T04:51:51.914060+0000","last_became_peered":"2024-03-29T04:51:51.914060+0000","last_unstale":"2024-03-29T04:52:33.686457+0000","last_undegraded":"2024-03-29T04:52:33.686457+0000","last_fullsized":"2024-03-29T04:52:33.686457+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:08:02.210421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686908+0000","last_change":"2024-03-29T04:51:50.891259+0000","last_active":"2024-03-29T04:52:33.686908+0000","last_peered":"2024-03-29T04:52:33.686908+0000","last_clean":"2024-03-29T04:52:33.686908+0000","last_became_active":"2024-03-29T04:51:50.890952+0000","last_became_peered":"2024-03-29T04:51:50.890952+0000","last_unstale":"2024-03-29T04:52:33.686908+0000","last_undegraded":"2024-03-29T04:52:33.686908+0000","last_fullsized":"2024-03-29T04:52:33.686908+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:01:46.663624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689217+0000","last_change":"2024-03-29T04:51:51.345846+0000","last_active":"2024-03-29T04:52:33.689217+0000","last_peered":"2024-03-29T04:52:33.689217+0000","last_clean":"2024-03-29T04:52:33.689217+0000","last_became_active":"2024-03-29T04:51:51.345551+0000","last_became_peered":"2024-03-29T04:51:51.345551+0000","last_unstale":"2024-03-29T04:52:33.689217+0000","last_undegraded":"2024-03-29T04:52:33.689217+0000","last_fullsized":"2024-03-29T04:52:33.689217+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:56:13.597807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690090+0000","last_change":"2024-03-29T04:51:50.895069+0000","last_active":"2024-03-29T04:52:33.690090+0000","last_peered":"2024-03-29T04:52:33.690090+0000","last_clean":"2024-03-29T04:52:33.690090+0000","last_became_active":"2024-03-29T04:51:50.894761+0000","last_became_peered":"2024-03-29T04:51:50.894761+0000","last_unstale":"2024-03-29T04:52:33.690090+0000","last_undegraded":"2024-03-29T04:52:33.690090+0000","last_fullsized":"2024-03-29T04:52:33.690090+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:52:53.436984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689596+0000","last_change":"2024-03-29T04:51:51.345898+0000","last_active":"2024-03-29T04:52:33.689596+0000","last_peered":"2024-03-29T04:52:33.689596+0000","last_clean":"2024-03-29T04:52:33.689596+0000","last_became_active":"2024-03-29T04:51:51.345579+0000","last_became_peered":"2024-03-29T04:51:51.345579+0000","last_unstale":"2024-03-29T04:52:33.689596+0000","last_undegraded":"2024-03-29T04:52:33.689596+0000","last_fullsized":"2024-03-29T04:52:33.689596+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:07:35.976888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689960+0000","last_change":"2024-03-29T04:51:50.892033+0000","last_active":"2024-03-29T04:52:33.689960+0000","last_peered":"2024-03-29T04:52:33.689960+0000","last_clean":"2024-03-29T04:52:33.689960+0000","last_became_active":"2024-03-29T04:51:50.890155+0000","last_became_peered":"2024-03-29T04:51:50.890155+0000","last_unstale":"2024-03-29T04:52:33.689960+0000","last_undegraded":"2024-03-29T04:52:33.689960+0000","last_fullsized":"2024-03-29T04:52:33.689960+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:58:23.398616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689094+0000","last_change":"2024-03-29T04:51:51.915466+0000","last_active":"2024-03-29T04:52:33.689094+0000","last_peered":"2024-03-29T04:52:33.689094+0000","last_clean":"2024-03-29T04:52:33.689094+0000","last_became_active":"2024-03-29T04:51:51.915121+0000","last_became_peered":"2024-03-29T04:51:51.915121+0000","last_unstale":"2024-03-29T04:52:33.689094+0000","last_undegraded":"2024-03-29T04:52:33.689094+0000","last_fullsized":"2024-03-29T04:52:33.689094+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:00:34.141526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.691440+0000","last_change":"2024-03-29T04:51:50.893229+0000","last_active":"2024-03-29T04:52:33.691440+0000","last_peered":"2024-03-29T04:52:33.691440+0000","last_clean":"2024-03-29T04:52:33.691440+0000","last_became_active":"2024-03-29T04:51:50.891671+0000","last_became_peered":"2024-03-29T04:51:50.891671+0000","last_unstale":"2024-03-29T04:52:33.691440+0000","last_undegraded":"2024-03-29T04:52:33.691440+0000","last_fullsized":"2024-03-29T04:52:33.691440+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:59:18.841471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.688976+0000","last_change":"2024-03-29T04:51:50.904139+0000","last_active":"2024-03-29T04:52:33.688976+0000","last_peered":"2024-03-29T04:52:33.688976+0000","last_clean":"2024-03-29T04:52:33.688976+0000","last_became_active":"2024-03-29T04:51:50.903780+0000","last_became_peered":"2024-03-29T04:51:50.903780+0000","last_unstale":"2024-03-29T04:52:33.688976+0000","last_undegraded":"2024-03-29T04:52:33.688976+0000","last_fullsized":"2024-03-29T04:52:33.688976+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:31:58.484667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.692078+0000","last_change":"2024-03-29T04:51:50.891247+0000","last_active":"2024-03-29T04:52:33.692078+0000","last_peered":"2024-03-29T04:52:33.692078+0000","last_clean":"2024-03-29T04:52:33.692078+0000","last_became_active":"2024-03-29T04:51:50.890257+0000","last_became_peered":"2024-03-29T04:51:50.890257+0000","last_unstale":"2024-03-29T04:52:33.692078+0000","last_undegraded":"2024-03-29T04:52:33.692078+0000","last_fullsized":"2024-03-29T04:52:33.692078+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:30:04.320976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913110+0000","last_change":"2024-03-29T04:50:39.046369+0000","last_active":"2024-03-29T04:52:33.913110+0000","last_peered":"2024-03-29T04:52:33.913110+0000","last_clean":"2024-03-29T04:52:33.913110+0000","last_became_active":"2024-03-29T04:50:39.046086+0000","last_became_peered":"2024-03-29T04:50:39.046086+0000","last_unstale":"2024-03-29T04:52:33.913110+0000","last_undegraded":"2024-03-29T04:52:33.913110+0000","last_fullsized":"2024-03-29T04:52:33.913110+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:26:33.515943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686221+0000","last_change":"2024-03-29T04:51:52.387075+0000","last_active":"2024-03-29T04:52:33.686221+0000","last_peered":"2024-03-29T04:52:33.686221+0000","last_clean":"2024-03-29T04:52:33.686221+0000","last_became_active":"2024-03-29T04:51:52.386763+0000","last_became_peered":"2024-03-29T04:51:52.386763+0000","last_unstale":"2024-03-29T04:52:33.686221+0000","last_undegraded":"2024-03-29T04:52:33.686221+0000","last_fullsized":"2024-03-29T04:52:33.686221+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:31:09.993264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690167+0000","last_change":"2024-03-29T04:50:39.220266+0000","last_active":"2024-03-29T04:52:33.690167+0000","last_peered":"2024-03-29T04:52:33.690167+0000","last_clean":"2024-03-29T04:52:33.690167+0000","last_became_active":"2024-03-29T04:50:39.219919+0000","last_became_peered":"2024-03-29T04:50:39.219919+0000","last_unstale":"2024-03-29T04:52:33.690167+0000","last_undegraded":"2024-03-29T04:52:33.690167+0000","last_fullsized":"2024-03-29T04:52:33.690167+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:16:10.835704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.687043+0000","last_change":"2024-03-29T04:51:50.892648+0000","last_active":"2024-03-29T04:52:33.687043+0000","last_peered":"2024-03-29T04:52:33.687043+0000","last_clean":"2024-03-29T04:52:33.687043+0000","last_became_active":"2024-03-29T04:51:50.892381+0000","last_became_peered":"2024-03-29T04:51:50.892381+0000","last_unstale":"2024-03-29T04:52:33.687043+0000","last_undegraded":"2024-03-29T04:52:33.687043+0000","last_fullsized":"2024-03-29T04:52:33.687043+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:25:06.918009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.691341+0000","last_change":"2024-03-29T04:51:52.386981+0000","last_active":"2024-03-29T04:52:33.691341+0000","last_peered":"2024-03-29T04:52:33.691341+0000","last_clean":"2024-03-29T04:52:33.691341+0000","last_became_active":"2024-03-29T04:51:52.386716+0000","last_became_peered":"2024-03-29T04:51:52.386716+0000","last_unstale":"2024-03-29T04:52:33.691341+0000","last_undegraded":"2024-03-29T04:52:33.691341+0000","last_fullsized":"2024-03-29T04:52:33.691341+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:09:33.409779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690145+0000","last_change":"2024-03-29T04:51:50.889224+0000","last_active":"2024-03-29T04:52:33.690145+0000","last_peered":"2024-03-29T04:52:33.690145+0000","last_clean":"2024-03-29T04:52:33.690145+0000","last_became_active":"2024-03-29T04:51:50.888933+0000","last_became_peered":"2024-03-29T04:51:50.888933+0000","last_unstale":"2024-03-29T04:52:33.690145+0000","last_undegraded":"2024-03-29T04:52:33.690145+0000","last_fullsized":"2024-03-29T04:52:33.690145+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:41:08.015792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913496+0000","last_change":"2024-03-29T04:50:39.219980+0000","last_active":"2024-03-29T04:52:33.913496+0000","last_peered":"2024-03-29T04:52:33.913496+0000","last_clean":"2024-03-29T04:52:33.913496+0000","last_became_active":"2024-03-29T04:50:39.219627+0000","last_became_peered":"2024-03-29T04:50:39.219627+0000","last_unstale":"2024-03-29T04:52:33.913496+0000","last_undegraded":"2024-03-29T04:52:33.913496+0000","last_fullsized":"2024-03-29T04:52:33.913496+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:56:32.992193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-29T04:52:34.965896+0000","last_change":"2024-03-29T04:52:34.965896+0000","last_active":"2024-03-29T04:52:34.965896+0000","last_peered":"2024-03-29T04:52:34.965896+0000","last_clean":"2024-03-29T04:52:34.965896+0000","last_became_active":"2024-03-29T04:52:34.965513+0000","last_became_peered":"2024-03-29T04:52:34.965513+0000","last_unstale":"2024-03-29T04:52:34.965896+0000","last_undegraded":"2024-03-29T04:52:34.965896+0000","last_fullsized":"2024-03-29T04:52:34.965896+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:09:46.323541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.691301+0000","last_change":"2024-03-29T04:51:50.890387+0000","last_active":"2024-03-29T04:52:33.691301+0000","last_peered":"2024-03-29T04:52:33.691301+0000","last_clean":"2024-03-29T04:52:33.691301+0000","last_became_active":"2024-03-29T04:51:50.890015+0000","last_became_peered":"2024-03-29T04:51:50.890015+0000","last_unstale":"2024-03-29T04:52:33.691301+0000","last_undegraded":"2024-03-29T04:52:33.691301+0000","last_fullsized":"2024-03-29T04:52:33.691301+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:40:09.378351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913157+0000","last_change":"2024-03-29T04:50:39.149486+0000","last_active":"2024-03-29T04:52:33.913157+0000","last_peered":"2024-03-29T04:52:33.913157+0000","last_clean":"2024-03-29T04:52:33.913157+0000","last_became_active":"2024-03-29T04:50:39.149236+0000","last_became_peered":"2024-03-29T04:50:39.149236+0000","last_unstale":"2024-03-29T04:52:33.913157+0000","last_undegraded":"2024-03-29T04:52:33.913157+0000","last_fullsized":"2024-03-29T04:52:33.913157+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:14:50.050422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913220+0000","last_change":"2024-03-29T04:51:51.765782+0000","last_active":"2024-03-29T04:52:33.913220+0000","last_peered":"2024-03-29T04:52:33.913220+0000","last_clean":"2024-03-29T04:52:33.913220+0000","last_became_active":"2024-03-29T04:51:51.765491+0000","last_became_peered":"2024-03-29T04:51:51.765491+0000","last_unstale":"2024-03-29T04:52:33.913220+0000","last_undegraded":"2024-03-29T04:52:33.913220+0000","last_fullsized":"2024-03-29T04:52:33.913220+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:18:02.508416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686587+0000","last_change":"2024-03-29T04:51:50.892178+0000","last_active":"2024-03-29T04:52:33.686587+0000","last_peered":"2024-03-29T04:52:33.686587+0000","last_clean":"2024-03-29T04:52:33.686587+0000","last_became_active":"2024-03-29T04:51:50.891686+0000","last_became_peered":"2024-03-29T04:51:50.891686+0000","last_unstale":"2024-03-29T04:52:33.686587+0000","last_undegraded":"2024-03-29T04:52:33.686587+0000","last_fullsized":"2024-03-29T04:52:33.686587+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:24:14.883625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":278,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913049+0000","last_change":"2024-03-29T04:50:05.432965+0000","last_active":"2024-03-29T04:52:33.913049+0000","last_peered":"2024-03-29T04:52:33.913049+0000","last_clean":"2024-03-29T04:52:33.913049+0000","last_became_active":"2024-03-29T04:50:05.432530+0000","last_became_peered":"2024-03-29T04:50:05.432530+0000","last_unstale":"2024-03-29T04:52:33.913049+0000","last_undegraded":"2024-03-29T04:52:33.913049+0000","last_fullsized":"2024-03-29T04:52:33.913049+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:48:01.601528+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:48:01.601528+0000","last_clean_scrub_stamp":"2024-03-29T04:48:01.601528+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:56:20.936763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689817+0000","last_change":"2024-03-29T04:50:39.045035+0000","last_active":"2024-03-29T04:52:33.689817+0000","last_peered":"2024-03-29T04:52:33.689817+0000","last_clean":"2024-03-29T04:52:33.689817+0000","last_became_active":"2024-03-29T04:50:39.044724+0000","last_became_peered":"2024-03-29T04:50:39.044724+0000","last_unstale":"2024-03-29T04:52:33.689817+0000","last_undegraded":"2024-03-29T04:52:33.689817+0000","last_fullsized":"2024-03-29T04:52:33.689817+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:44:08.632467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689771+0000","last_change":"2024-03-29T04:51:51.281211+0000","last_active":"2024-03-29T04:52:33.689771+0000","last_peered":"2024-03-29T04:52:33.689771+0000","last_clean":"2024-03-29T04:52:33.689771+0000","last_became_active":"2024-03-29T04:51:51.280893+0000","last_became_peered":"2024-03-29T04:51:51.280893+0000","last_unstale":"2024-03-29T04:52:33.689771+0000","last_undegraded":"2024-03-29T04:52:33.689771+0000","last_fullsized":"2024-03-29T04:52:33.689771+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:37:24.853594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.688835+0000","last_change":"2024-03-29T04:51:50.892366+0000","last_active":"2024-03-29T04:52:33.688835+0000","last_peered":"2024-03-29T04:52:33.688835+0000","last_clean":"2024-03-29T04:52:33.688835+0000","last_became_active":"2024-03-29T04:51:50.892101+0000","last_became_peered":"2024-03-29T04:51:50.892101+0000","last_unstale":"2024-03-29T04:52:33.688835+0000","last_undegraded":"2024-03-29T04:52:33.688835+0000","last_fullsized":"2024-03-29T04:52:33.688835+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:52:18.835709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689033+0000","last_change":"2024-03-29T04:50:39.246218+0000","last_active":"2024-03-29T04:52:33.689033+0000","last_peered":"2024-03-29T04:52:33.689033+0000","last_clean":"2024-03-29T04:52:33.689033+0000","last_became_active":"2024-03-29T04:50:39.245880+0000","last_became_peered":"2024-03-29T04:50:39.245880+0000","last_unstale":"2024-03-29T04:52:33.689033+0000","last_undegraded":"2024-03-29T04:52:33.689033+0000","last_fullsized":"2024-03-29T04:52:33.689033+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:17:07.467340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-29T04:52:34.965706+0000","last_change":"2024-03-29T04:52:34.965706+0000","last_active":"2024-03-29T04:52:34.965706+0000","last_peered":"2024-03-29T04:52:34.965706+0000","last_clean":"2024-03-29T04:52:34.965706+0000","last_became_active":"2024-03-29T04:52:34.965452+0000","last_became_peered":"2024-03-29T04:52:34.965452+0000","last_unstale":"2024-03-29T04:52:34.965706+0000","last_undegraded":"2024-03-29T04:52:34.965706+0000","last_fullsized":"2024-03-29T04:52:34.965706+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:35:28.368820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689231+0000","last_change":"2024-03-29T04:51:50.891485+0000","last_active":"2024-03-29T04:52:33.689231+0000","last_peered":"2024-03-29T04:52:33.689231+0000","last_clean":"2024-03-29T04:52:33.689231+0000","last_became_active":"2024-03-29T04:51:50.891222+0000","last_became_peered":"2024-03-29T04:51:50.891222+0000","last_unstale":"2024-03-29T04:52:33.689231+0000","last_undegraded":"2024-03-29T04:52:33.689231+0000","last_fullsized":"2024-03-29T04:52:33.689231+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:51:11.170455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689479+0000","last_change":"2024-03-29T04:50:39.044530+0000","last_active":"2024-03-29T04:52:33.689479+0000","last_peered":"2024-03-29T04:52:33.689479+0000","last_clean":"2024-03-29T04:52:33.689479+0000","last_became_active":"2024-03-29T04:50:39.044136+0000","last_became_peered":"2024-03-29T04:50:39.044136+0000","last_unstale":"2024-03-29T04:52:33.689479+0000","last_undegraded":"2024-03-29T04:52:33.689479+0000","last_fullsized":"2024-03-29T04:52:33.689479+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:32:12.577651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.688856+0000","last_change":"2024-03-29T04:51:50.904279+0000","last_active":"2024-03-29T04:52:33.688856+0000","last_peered":"2024-03-29T04:52:33.688856+0000","last_clean":"2024-03-29T04:52:33.688856+0000","last_became_active":"2024-03-29T04:51:50.904049+0000","last_became_peered":"2024-03-29T04:51:50.904049+0000","last_unstale":"2024-03-29T04:52:33.688856+0000","last_undegraded":"2024-03-29T04:52:33.688856+0000","last_fullsized":"2024-03-29T04:52:33.688856+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:44:42.492894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689998+0000","last_change":"2024-03-29T04:51:50.887938+0000","last_active":"2024-03-29T04:52:33.689998+0000","last_peered":"2024-03-29T04:52:33.689998+0000","last_clean":"2024-03-29T04:52:33.689998+0000","last_became_active":"2024-03-29T04:51:50.887501+0000","last_became_peered":"2024-03-29T04:51:50.887501+0000","last_unstale":"2024-03-29T04:52:33.689998+0000","last_undegraded":"2024-03-29T04:52:33.689998+0000","last_fullsized":"2024-03-29T04:52:33.689998+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:27:22.602641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690051+0000","last_change":"2024-03-29T04:50:39.149460+0000","last_active":"2024-03-29T04:52:33.690051+0000","last_peered":"2024-03-29T04:52:33.690051+0000","last_clean":"2024-03-29T04:52:33.690051+0000","last_became_active":"2024-03-29T04:50:39.149152+0000","last_became_peered":"2024-03-29T04:50:39.149152+0000","last_unstale":"2024-03-29T04:52:33.690051+0000","last_undegraded":"2024-03-29T04:52:33.690051+0000","last_fullsized":"2024-03-29T04:52:33.690051+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:05:37.697946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689700+0000","last_change":"2024-03-29T04:51:51.346261+0000","last_active":"2024-03-29T04:52:33.689700+0000","last_peered":"2024-03-29T04:52:33.689700+0000","last_clean":"2024-03-29T04:52:33.689700+0000","last_became_active":"2024-03-29T04:51:51.345637+0000","last_became_peered":"2024-03-29T04:51:51.345637+0000","last_unstale":"2024-03-29T04:52:33.689700+0000","last_undegraded":"2024-03-29T04:52:33.689700+0000","last_fullsized":"2024-03-29T04:52:33.689700+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:50:03.025362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686221+0000","last_change":"2024-03-29T04:51:50.890374+0000","last_active":"2024-03-29T04:52:33.686221+0000","last_peered":"2024-03-29T04:52:33.686221+0000","last_clean":"2024-03-29T04:52:33.686221+0000","last_became_active":"2024-03-29T04:51:50.890087+0000","last_became_peered":"2024-03-29T04:51:50.890087+0000","last_unstale":"2024-03-29T04:52:33.686221+0000","last_undegraded":"2024-03-29T04:52:33.686221+0000","last_fullsized":"2024-03-29T04:52:33.686221+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:58:26.599714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690156+0000","last_change":"2024-03-29T04:51:50.905623+0000","last_active":"2024-03-29T04:52:33.690156+0000","last_peered":"2024-03-29T04:52:33.690156+0000","last_clean":"2024-03-29T04:52:33.690156+0000","last_became_active":"2024-03-29T04:51:50.905346+0000","last_became_peered":"2024-03-29T04:51:50.905346+0000","last_unstale":"2024-03-29T04:52:33.690156+0000","last_undegraded":"2024-03-29T04:52:33.690156+0000","last_fullsized":"2024-03-29T04:52:33.690156+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:05:20.465451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689027+0000","last_change":"2024-03-29T04:51:50.892490+0000","last_active":"2024-03-29T04:52:33.689027+0000","last_peered":"2024-03-29T04:52:33.689027+0000","last_clean":"2024-03-29T04:52:33.689027+0000","last_became_active":"2024-03-29T04:51:50.892051+0000","last_became_peered":"2024-03-29T04:51:50.892051+0000","last_unstale":"2024-03-29T04:52:33.689027+0000","last_undegraded":"2024-03-29T04:52:33.689027+0000","last_fullsized":"2024-03-29T04:52:33.689027+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:44:19.582886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690536+0000","last_change":"2024-03-29T04:51:51.234266+0000","last_active":"2024-03-29T04:52:33.690536+0000","last_peered":"2024-03-29T04:52:33.690536+0000","last_clean":"2024-03-29T04:52:33.690536+0000","last_became_active":"2024-03-29T04:51:51.233943+0000","last_became_peered":"2024-03-29T04:51:51.233943+0000","last_unstale":"2024-03-29T04:52:33.690536+0000","last_undegraded":"2024-03-29T04:52:33.690536+0000","last_fullsized":"2024-03-29T04:52:33.690536+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:30:49.735372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913057+0000","last_change":"2024-03-29T04:51:50.897624+0000","last_active":"2024-03-29T04:52:33.913057+0000","last_peered":"2024-03-29T04:52:33.913057+0000","last_clean":"2024-03-29T04:52:33.913057+0000","last_became_active":"2024-03-29T04:51:50.897348+0000","last_became_peered":"2024-03-29T04:51:50.897348+0000","last_unstale":"2024-03-29T04:52:33.913057+0000","last_undegraded":"2024-03-29T04:52:33.913057+0000","last_fullsized":"2024-03-29T04:52:33.913057+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:16:56.513567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690100+0000","last_change":"2024-03-29T04:51:51.346008+0000","last_active":"2024-03-29T04:52:33.690100+0000","last_peered":"2024-03-29T04:52:33.690100+0000","last_clean":"2024-03-29T04:52:33.690100+0000","last_became_active":"2024-03-29T04:51:51.345554+0000","last_became_peered":"2024-03-29T04:51:51.345554+0000","last_unstale":"2024-03-29T04:52:33.690100+0000","last_undegraded":"2024-03-29T04:52:33.690100+0000","last_fullsized":"2024-03-29T04:52:33.690100+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:04:57.630878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686642+0000","last_change":"2024-03-29T04:51:50.894562+0000","last_active":"2024-03-29T04:52:33.686642+0000","last_peered":"2024-03-29T04:52:33.686642+0000","last_clean":"2024-03-29T04:52:33.686642+0000","last_became_active":"2024-03-29T04:51:50.894244+0000","last_became_peered":"2024-03-29T04:51:50.894244+0000","last_unstale":"2024-03-29T04:52:33.686642+0000","last_undegraded":"2024-03-29T04:52:33.686642+0000","last_fullsized":"2024-03-29T04:52:33.686642+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:41:22.958621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.691530+0000","last_change":"2024-03-29T04:51:52.096173+0000","last_active":"2024-03-29T04:52:33.691530+0000","last_peered":"2024-03-29T04:52:33.691530+0000","last_clean":"2024-03-29T04:52:33.691530+0000","last_became_active":"2024-03-29T04:51:52.095857+0000","last_became_peered":"2024-03-29T04:51:52.095857+0000","last_unstale":"2024-03-29T04:52:33.691530+0000","last_undegraded":"2024-03-29T04:52:33.691530+0000","last_fullsized":"2024-03-29T04:52:33.691530+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:47:28.087749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689849+0000","last_change":"2024-03-29T04:51:50.889041+0000","last_active":"2024-03-29T04:52:33.689849+0000","last_peered":"2024-03-29T04:52:33.689849+0000","last_clean":"2024-03-29T04:52:33.689849+0000","last_became_active":"2024-03-29T04:51:50.888733+0000","last_became_peered":"2024-03-29T04:51:50.888733+0000","last_unstale":"2024-03-29T04:52:33.689849+0000","last_undegraded":"2024-03-29T04:52:33.689849+0000","last_fullsized":"2024-03-29T04:52:33.689849+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:06:45.856873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.912879+0000","last_change":"2024-03-29T04:51:51.764934+0000","last_active":"2024-03-29T04:52:33.912879+0000","last_peered":"2024-03-29T04:52:33.912879+0000","last_clean":"2024-03-29T04:52:33.912879+0000","last_became_active":"2024-03-29T04:51:51.764617+0000","last_became_peered":"2024-03-29T04:51:51.764617+0000","last_unstale":"2024-03-29T04:52:33.912879+0000","last_undegraded":"2024-03-29T04:52:33.912879+0000","last_fullsized":"2024-03-29T04:52:33.912879+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:33:50.709751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913643+0000","last_change":"2024-03-29T04:51:50.896476+0000","last_active":"2024-03-29T04:52:33.913643+0000","last_peered":"2024-03-29T04:52:33.913643+0000","last_clean":"2024-03-29T04:52:33.913643+0000","last_became_active":"2024-03-29T04:51:50.895258+0000","last_became_peered":"2024-03-29T04:51:50.895258+0000","last_unstale":"2024-03-29T04:52:33.913643+0000","last_undegraded":"2024-03-29T04:52:33.913643+0000","last_fullsized":"2024-03-29T04:52:33.913643+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:24:42.583134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.688855+0000","last_change":"2024-03-29T04:51:51.345825+0000","last_active":"2024-03-29T04:52:33.688855+0000","last_peered":"2024-03-29T04:52:33.688855+0000","last_clean":"2024-03-29T04:52:33.688855+0000","last_became_active":"2024-03-29T04:51:51.345484+0000","last_became_peered":"2024-03-29T04:51:51.345484+0000","last_unstale":"2024-03-29T04:52:33.688855+0000","last_undegraded":"2024-03-29T04:52:33.688855+0000","last_fullsized":"2024-03-29T04:52:33.688855+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:12:47.292042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913636+0000","last_change":"2024-03-29T04:51:50.890021+0000","last_active":"2024-03-29T04:52:33.913636+0000","last_peered":"2024-03-29T04:52:33.913636+0000","last_clean":"2024-03-29T04:52:33.913636+0000","last_became_active":"2024-03-29T04:51:50.889698+0000","last_became_peered":"2024-03-29T04:51:50.889698+0000","last_unstale":"2024-03-29T04:52:33.913636+0000","last_undegraded":"2024-03-29T04:52:33.913636+0000","last_fullsized":"2024-03-29T04:52:33.913636+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:49:55.193916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689466+0000","last_change":"2024-03-29T04:51:51.764872+0000","last_active":"2024-03-29T04:52:33.689466+0000","last_peered":"2024-03-29T04:52:33.689466+0000","last_clean":"2024-03-29T04:52:33.689466+0000","last_became_active":"2024-03-29T04:51:51.764554+0000","last_became_peered":"2024-03-29T04:51:51.764554+0000","last_unstale":"2024-03-29T04:52:33.689466+0000","last_undegraded":"2024-03-29T04:52:33.689466+0000","last_fullsized":"2024-03-29T04:52:33.689466+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:09:07.383326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913717+0000","last_change":"2024-03-29T04:51:50.895042+0000","last_active":"2024-03-29T04:52:33.913717+0000","last_peered":"2024-03-29T04:52:33.913717+0000","last_clean":"2024-03-29T04:52:33.913717+0000","last_became_active":"2024-03-29T04:51:50.894731+0000","last_became_peered":"2024-03-29T04:51:50.894731+0000","last_unstale":"2024-03-29T04:52:33.913717+0000","last_undegraded":"2024-03-29T04:52:33.913717+0000","last_fullsized":"2024-03-29T04:52:33.913717+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:30:15.164924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686937+0000","last_change":"2024-03-29T04:51:52.389279+0000","last_active":"2024-03-29T04:52:33.686937+0000","last_peered":"2024-03-29T04:52:33.686937+0000","last_clean":"2024-03-29T04:52:33.686937+0000","last_became_active":"2024-03-29T04:51:52.388996+0000","last_became_peered":"2024-03-29T04:51:52.388996+0000","last_unstale":"2024-03-29T04:52:33.686937+0000","last_undegraded":"2024-03-29T04:52:33.686937+0000","last_fullsized":"2024-03-29T04:52:33.686937+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:41:32.316840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913170+0000","last_change":"2024-03-29T04:51:50.892088+0000","last_active":"2024-03-29T04:52:33.913170+0000","last_peered":"2024-03-29T04:52:33.913170+0000","last_clean":"2024-03-29T04:52:33.913170+0000","last_became_active":"2024-03-29T04:51:50.891771+0000","last_became_peered":"2024-03-29T04:51:50.891771+0000","last_unstale":"2024-03-29T04:52:33.913170+0000","last_undegraded":"2024-03-29T04:52:33.913170+0000","last_fullsized":"2024-03-29T04:52:33.913170+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:29:38.982286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.691964+0000","last_change":"2024-03-29T04:51:52.388437+0000","last_active":"2024-03-29T04:52:33.691964+0000","last_peered":"2024-03-29T04:52:33.691964+0000","last_clean":"2024-03-29T04:52:33.691964+0000","last_became_active":"2024-03-29T04:51:52.388167+0000","last_became_peered":"2024-03-29T04:51:52.388167+0000","last_unstale":"2024-03-29T04:52:33.691964+0000","last_undegraded":"2024-03-29T04:52:33.691964+0000","last_fullsized":"2024-03-29T04:52:33.691964+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:09:14.684669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.912865+0000","last_change":"2024-03-29T04:51:50.897482+0000","last_active":"2024-03-29T04:52:33.912865+0000","last_peered":"2024-03-29T04:52:33.912865+0000","last_clean":"2024-03-29T04:52:33.912865+0000","last_became_active":"2024-03-29T04:51:50.896991+0000","last_became_peered":"2024-03-29T04:51:50.896991+0000","last_unstale":"2024-03-29T04:52:33.912865+0000","last_undegraded":"2024-03-29T04:52:33.912865+0000","last_fullsized":"2024-03-29T04:52:33.912865+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:53:18.957886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689899+0000","last_change":"2024-03-29T04:51:51.766321+0000","last_active":"2024-03-29T04:52:33.689899+0000","last_peered":"2024-03-29T04:52:33.689899+0000","last_clean":"2024-03-29T04:52:33.689899+0000","last_became_active":"2024-03-29T04:51:51.765964+0000","last_became_peered":"2024-03-29T04:51:51.765964+0000","last_unstale":"2024-03-29T04:52:33.689899+0000","last_undegraded":"2024-03-29T04:52:33.689899+0000","last_fullsized":"2024-03-29T04:52:33.689899+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:16:59.703129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.691406+0000","last_change":"2024-03-29T04:51:50.889856+0000","last_active":"2024-03-29T04:52:33.691406+0000","last_peered":"2024-03-29T04:52:33.691406+0000","last_clean":"2024-03-29T04:52:33.691406+0000","last_became_active":"2024-03-29T04:51:50.889578+0000","last_became_peered":"2024-03-29T04:51:50.889578+0000","last_unstale":"2024-03-29T04:52:33.691406+0000","last_undegraded":"2024-03-29T04:52:33.691406+0000","last_fullsized":"2024-03-29T04:52:33.691406+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:39:31.797252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686773+0000","last_change":"2024-03-29T04:51:52.388513+0000","last_active":"2024-03-29T04:52:33.686773+0000","last_peered":"2024-03-29T04:52:33.686773+0000","last_clean":"2024-03-29T04:52:33.686773+0000","last_became_active":"2024-03-29T04:51:52.387739+0000","last_became_peered":"2024-03-29T04:51:52.387739+0000","last_unstale":"2024-03-29T04:52:33.686773+0000","last_undegraded":"2024-03-29T04:52:33.686773+0000","last_fullsized":"2024-03-29T04:52:33.686773+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:38:26.826741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913367+0000","last_change":"2024-03-29T04:51:50.896025+0000","last_active":"2024-03-29T04:52:33.913367+0000","last_peered":"2024-03-29T04:52:33.913367+0000","last_clean":"2024-03-29T04:52:33.913367+0000","last_became_active":"2024-03-29T04:51:50.895639+0000","last_became_peered":"2024-03-29T04:51:50.895639+0000","last_unstale":"2024-03-29T04:52:33.913367+0000","last_undegraded":"2024-03-29T04:52:33.913367+0000","last_fullsized":"2024-03-29T04:52:33.913367+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:42:09.132536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689777+0000","last_change":"2024-03-29T04:51:50.907295+0000","last_active":"2024-03-29T04:52:33.689777+0000","last_peered":"2024-03-29T04:52:33.689777+0000","last_clean":"2024-03-29T04:52:33.689777+0000","last_became_active":"2024-03-29T04:51:50.906975+0000","last_became_peered":"2024-03-29T04:51:50.906975+0000","last_unstale":"2024-03-29T04:52:33.689777+0000","last_undegraded":"2024-03-29T04:52:33.689777+0000","last_fullsized":"2024-03-29T04:52:33.689777+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:47:28.195544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690080+0000","last_change":"2024-03-29T04:51:50.890463+0000","last_active":"2024-03-29T04:52:33.690080+0000","last_peered":"2024-03-29T04:52:33.690080+0000","last_clean":"2024-03-29T04:52:33.690080+0000","last_became_active":"2024-03-29T04:51:50.890171+0000","last_became_peered":"2024-03-29T04:51:50.890171+0000","last_unstale":"2024-03-29T04:52:33.690080+0000","last_undegraded":"2024-03-29T04:52:33.690080+0000","last_fullsized":"2024-03-29T04:52:33.690080+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:08:23.559107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.688767+0000","last_change":"2024-03-29T04:51:51.234068+0000","last_active":"2024-03-29T04:52:33.688767+0000","last_peered":"2024-03-29T04:52:33.688767+0000","last_clean":"2024-03-29T04:52:33.688767+0000","last_became_active":"2024-03-29T04:51:51.233793+0000","last_became_peered":"2024-03-29T04:51:51.233793+0000","last_unstale":"2024-03-29T04:52:33.688767+0000","last_undegraded":"2024-03-29T04:52:33.688767+0000","last_fullsized":"2024-03-29T04:52:33.688767+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:29:37.505596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913490+0000","last_change":"2024-03-29T04:51:50.889057+0000","last_active":"2024-03-29T04:52:33.913490+0000","last_peered":"2024-03-29T04:52:33.913490+0000","last_clean":"2024-03-29T04:52:33.913490+0000","last_became_active":"2024-03-29T04:51:50.888695+0000","last_became_peered":"2024-03-29T04:51:50.888695+0000","last_unstale":"2024-03-29T04:52:33.913490+0000","last_undegraded":"2024-03-29T04:52:33.913490+0000","last_fullsized":"2024-03-29T04:52:33.913490+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:12:03.714559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686467+0000","last_change":"2024-03-29T04:51:52.388562+0000","last_active":"2024-03-29T04:52:33.686467+0000","last_peered":"2024-03-29T04:52:33.686467+0000","last_clean":"2024-03-29T04:52:33.686467+0000","last_became_active":"2024-03-29T04:51:52.388312+0000","last_became_peered":"2024-03-29T04:51:52.388312+0000","last_unstale":"2024-03-29T04:52:33.686467+0000","last_undegraded":"2024-03-29T04:52:33.686467+0000","last_fullsized":"2024-03-29T04:52:33.686467+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:26:24.263187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690179+0000","last_change":"2024-03-29T04:51:50.891872+0000","last_active":"2024-03-29T04:52:33.690179+0000","last_peered":"2024-03-29T04:52:33.690179+0000","last_clean":"2024-03-29T04:52:33.690179+0000","last_became_active":"2024-03-29T04:51:50.891510+0000","last_became_peered":"2024-03-29T04:51:50.891510+0000","last_unstale":"2024-03-29T04:52:33.690179+0000","last_undegraded":"2024-03-29T04:52:33.690179+0000","last_fullsized":"2024-03-29T04:52:33.690179+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:18:15.904788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686318+0000","last_change":"2024-03-29T04:51:52.386593+0000","last_active":"2024-03-29T04:52:33.686318+0000","last_peered":"2024-03-29T04:52:33.686318+0000","last_clean":"2024-03-29T04:52:33.686318+0000","last_became_active":"2024-03-29T04:51:52.386289+0000","last_became_peered":"2024-03-29T04:51:52.386289+0000","last_unstale":"2024-03-29T04:52:33.686318+0000","last_undegraded":"2024-03-29T04:52:33.686318+0000","last_fullsized":"2024-03-29T04:52:33.686318+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:01:03.675008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.691661+0000","last_change":"2024-03-29T04:51:50.886309+0000","last_active":"2024-03-29T04:52:33.691661+0000","last_peered":"2024-03-29T04:52:33.691661+0000","last_clean":"2024-03-29T04:52:33.691661+0000","last_became_active":"2024-03-29T04:51:50.885946+0000","last_became_peered":"2024-03-29T04:51:50.885946+0000","last_unstale":"2024-03-29T04:52:33.691661+0000","last_undegraded":"2024-03-29T04:52:33.691661+0000","last_fullsized":"2024-03-29T04:52:33.691661+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:42:31.532903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.912995+0000","last_change":"2024-03-29T04:51:50.904596+0000","last_active":"2024-03-29T04:52:33.912995+0000","last_peered":"2024-03-29T04:52:33.912995+0000","last_clean":"2024-03-29T04:52:33.912995+0000","last_became_active":"2024-03-29T04:51:50.904303+0000","last_became_peered":"2024-03-29T04:51:50.904303+0000","last_unstale":"2024-03-29T04:52:33.912995+0000","last_undegraded":"2024-03-29T04:52:33.912995+0000","last_fullsized":"2024-03-29T04:52:33.912995+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:43:26.435518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686426+0000","last_change":"2024-03-29T04:51:50.892768+0000","last_active":"2024-03-29T04:52:33.686426+0000","last_peered":"2024-03-29T04:52:33.686426+0000","last_clean":"2024-03-29T04:52:33.686426+0000","last_became_active":"2024-03-29T04:51:50.891998+0000","last_became_peered":"2024-03-29T04:51:50.891998+0000","last_unstale":"2024-03-29T04:52:33.686426+0000","last_undegraded":"2024-03-29T04:52:33.686426+0000","last_fullsized":"2024-03-29T04:52:33.686426+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:23:22.977219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.692061+0000","last_change":"2024-03-29T04:51:52.388697+0000","last_active":"2024-03-29T04:52:33.692061+0000","last_peered":"2024-03-29T04:52:33.692061+0000","last_clean":"2024-03-29T04:52:33.692061+0000","last_became_active":"2024-03-29T04:51:52.388436+0000","last_became_peered":"2024-03-29T04:51:52.388436+0000","last_unstale":"2024-03-29T04:52:33.692061+0000","last_undegraded":"2024-03-29T04:52:33.692061+0000","last_fullsized":"2024-03-29T04:52:33.692061+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:14:53.361219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.688747+0000","last_change":"2024-03-29T04:51:50.890473+0000","last_active":"2024-03-29T04:52:33.688747+0000","last_peered":"2024-03-29T04:52:33.688747+0000","last_clean":"2024-03-29T04:52:33.688747+0000","last_became_active":"2024-03-29T04:51:50.889848+0000","last_became_peered":"2024-03-29T04:51:50.889848+0000","last_unstale":"2024-03-29T04:52:33.688747+0000","last_undegraded":"2024-03-29T04:52:33.688747+0000","last_fullsized":"2024-03-29T04:52:33.688747+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:37:30.285886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-29T04:52:35.115723+0000","last_change":"2024-03-29T04:52:35.115723+0000","last_active":"2024-03-29T04:52:35.115723+0000","last_peered":"2024-03-29T04:52:35.115723+0000","last_clean":"2024-03-29T04:52:35.115723+0000","last_became_active":"2024-03-29T04:52:35.115424+0000","last_became_peered":"2024-03-29T04:52:35.115424+0000","last_unstale":"2024-03-29T04:52:35.115723+0000","last_undegraded":"2024-03-29T04:52:35.115723+0000","last_fullsized":"2024-03-29T04:52:35.115723+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:29:02.714701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.688974+0000","last_change":"2024-03-29T04:51:50.901011+0000","last_active":"2024-03-29T04:52:33.688974+0000","last_peered":"2024-03-29T04:52:33.688974+0000","last_clean":"2024-03-29T04:52:33.688974+0000","last_became_active":"2024-03-29T04:51:50.900150+0000","last_became_peered":"2024-03-29T04:51:50.900150+0000","last_unstale":"2024-03-29T04:52:33.688974+0000","last_undegraded":"2024-03-29T04:52:33.688974+0000","last_fullsized":"2024-03-29T04:52:33.688974+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:48:34.206192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201316,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298672,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446480,"statfs":{"total":95995035648,"available":95689195520,"internally_reserved":0,"allocated":1343488,"data_stored":908282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54600000000000004,"15min":0.54700000000000004},"min":{"1min":0.40600000000000003,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.65200000000000002,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.53900000000000003,"15min":0.54300000000000004},"min":{"1min":0.33200000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.71899999999999997,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.501}]},{"osd":1,"last update":"Fri Mar 29 04:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55900000000000005,"15min":0.56000000000000005},"min":{"1min":0.371,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.76000000000000001,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.375},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.438}]},{"osd":2,"last update":"Fri Mar 29 04:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.54500000000000004,"15min":0.54900000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.75,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.41499999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.82999999999999996,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.56699999999999995}]},{"osd":3,"last update":"Fri Mar 29 04:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.40799999999999997,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.69899999999999995,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.40899999999999997,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.80200000000000005,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.47599999999999998}]},{"osd":4,"last update":"Fri Mar 29 04:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.48999999999999999,"15min":0.496},"min":{"1min":0.36799999999999999,"5min":0.313,"15min":0.313},"max":{"1min":0.58699999999999997,"5min":0.879,"15min":0.879},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.502,"15min":0.502},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.58799999999999997,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.58299999999999996}]},{"osd":5,"last update":"Fri Mar 29 04:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51100000000000001,"15min":0.51000000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.78700000000000003,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.53800000000000003,"15min":0.54000000000000004},"min":{"1min":0.40799999999999997,"5min":0.37,"15min":0.37},"max":{"1min":0.625,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.55000000000000004}]},{"osd":6,"last update":"Fri Mar 29 04:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.71199999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.55000000000000004,"15min":0.54800000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.74299999999999999,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.623}]}]},{"osd":6,"up_from":47,"seq":201863462954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298652,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446500,"statfs":{"total":95995035648,"available":95689216000,"internally_reserved":0,"allocated":1327104,"data_stored":907871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52700000000000002,"15min":0.53000000000000003},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52500000000000002,"15min":0.53600000000000003},"min":{"1min":0.41899999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.66000000000000003,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.46200000000000002}]},{"osd":1,"last update":"Fri Mar 29 04:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54000000000000004,"15min":0.54300000000000004},"min":{"1min":0.35199999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.55100000000000005,"15min":0.56200000000000006},"min":{"1min":0.41499999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73499999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.626}]},{"osd":2,"last update":"Fri Mar 29 04:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.53700000000000003,"15min":0.54400000000000004},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.53300000000000003,"15min":0.53400000000000003},"min":{"1min":0.39600000000000002,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.40500000000000003}]},{"osd":3,"last update":"Fri Mar 29 04:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.55300000000000005,"15min":0.55600000000000005},"min":{"1min":0.40200000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54600000000000004,"15min":0.55400000000000005},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.69899999999999995,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.54800000000000004}]},{"osd":4,"last update":"Fri Mar 29 04:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.45500000000000002,"15min":0.44600000000000001},"min":{"1min":0.313,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.69199999999999995,"5min":0.746,"15min":0.746},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.45600000000000002,"15min":0.42999999999999999},"min":{"1min":0.33600000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.67500000000000004,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.51100000000000001}]},{"osd":5,"last update":"Fri Mar 29 04:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.47299999999999998,"15min":0.443},"min":{"1min":0.36799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.67400000000000004,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.501,"15min":0.47099999999999997},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.73099999999999998,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.64100000000000001}]},{"osd":7,"last update":"Fri Mar 29 04:52:06 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.51200000000000001,"15min":0.50800000000000001},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.54100000000000004,"15min":0.53500000000000003},"min":{"1min":0.40400000000000003,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.67500000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":716800,"data_stored":315939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:51:45 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.35199999999999998,"15min":0.33700000000000002},"min":{"1min":0.251,"5min":0.216,"15min":0.216},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.372,"5min":0.38200000000000001,"15min":0.34999999999999998},"min":{"1min":0.23599999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.51100000000000001,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.28599999999999998}]},{"osd":2,"last update":"Fri Mar 29 04:52:05 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.39500000000000002,"15min":0.38300000000000001},"min":{"1min":0.27200000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.48699999999999999,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.44600000000000001,"15min":0.432},"min":{"1min":0.27700000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.58899999999999997,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.35799999999999998}]},{"osd":3,"last update":"Fri Mar 29 04:52:37 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.437,"15min":0.45000000000000001},"min":{"1min":0.28499999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.42499999999999999,"15min":0.441},"min":{"1min":0.27600000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.36799999999999999}]},{"osd":4,"last update":"Fri Mar 29 04:51:52 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.56599999999999995,"15min":0.57199999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.59099999999999997,"15min":0.59199999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.53800000000000003}]},{"osd":5,"last update":"Fri Mar 29 04:52:19 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57299999999999995,"15min":0.57099999999999995},"min":{"1min":0.40300000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.57399999999999995,"15min":0.57999999999999996},"min":{"1min":0.40500000000000003,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.73699999999999999,"5min":0.749,"15min":0.749},"last":0.51900000000000002}]},{"osd":6,"last update":"Fri Mar 29 04:52:39 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.57499999999999996,"15min":0.57899999999999996},"min":{"1min":0.42599999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.72299999999999998,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.441},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.59399999999999997,"15min":0.60399999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.498}]},{"osd":7,"last update":"Fri Mar 29 04:52:05 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.58399999999999996,"15min":0.58499999999999996},"min":{"1min":0.432,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.755,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.58199999999999996,"15min":0.58599999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.77900000000000003,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.58799999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298652,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446500,"statfs":{"total":95995035648,"available":95689216000,"internally_reserved":0,"allocated":1323008,"data_stored":907164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Mar 29 04:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.38200000000000001,"15min":0.36799999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.54900000000000004,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.38400000000000001,"15min":0.38200000000000001},"min":{"1min":0.25800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.54800000000000004,"5min":0.621,"15min":0.621},"last":0.42599999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:52:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40899999999999997,"15min":0.40300000000000002},"min":{"1min":0.28899999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.54400000000000004,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.40799999999999997,"15min":0.40699999999999997},"min":{"1min":0.28899999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.56000000000000005,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.40999999999999998}]},{"osd":3,"last update":"Fri Mar 29 04:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.439,"15min":0.45100000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.57599999999999996,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.41799999999999998,"15min":0.432},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.50700000000000001,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.47799999999999998}]},{"osd":4,"last update":"Fri Mar 29 04:51:54 2024","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.59499999999999997,"15min":0.59299999999999997},"min":{"1min":0.46500000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.57999999999999996,"15min":0.57499999999999996},"min":{"1min":0.40500000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.54900000000000004}]},{"osd":5,"last update":"Fri Mar 29 04:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.58499999999999996,"15min":0.58199999999999996},"min":{"1min":0.39200000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.56000000000000005,"15min":0.55400000000000005},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.61699999999999999}]},{"osd":6,"last update":"Fri Mar 29 04:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.55900000000000005,"15min":0.54800000000000004},"min":{"1min":0.38800000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.61299999999999999,"15min":0.61799999999999999},"min":{"1min":0.46200000000000002,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.60499999999999998}]},{"osd":7,"last update":"Fri Mar 29 04:52:04 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.434,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.80400000000000005,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.501},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.58799999999999997,"15min":0.58399999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77500000000000002,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.58399999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411390,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":729088,"data_stored":316808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:52:05 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.36299999999999999,"15min":0.35199999999999998},"min":{"1min":0.23400000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.48099999999999998,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.37},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.38800000000000001,"15min":0.36899999999999999},"min":{"1min":0.25800000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.35199999999999998}]},{"osd":1,"last update":"Fri Mar 29 04:52:05 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.38600000000000001,"15min":0.379},"min":{"1min":0.29099999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.502,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.40200000000000002,"15min":0.38900000000000001},"min":{"1min":0.26100000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.497,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.439}]},{"osd":3,"last update":"Fri Mar 29 04:52:32 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.53700000000000003,"5min":1.232,"15min":1.232},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.44700000000000001,"15min":0.47799999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.49399999999999999,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.33900000000000002}]},{"osd":4,"last update":"Fri Mar 29 04:51:50 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.59999999999999998,"15min":0.59599999999999997},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.82699999999999996,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.627,"15min":0.624},"min":{"1min":0.44800000000000001,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.79700000000000004,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.49199999999999999}]},{"osd":5,"last update":"Fri Mar 29 04:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59299999999999997,"15min":0.60299999999999998},"min":{"1min":0.42499999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.622,"15min":0.63700000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.84099999999999997,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.56899999999999995}]},{"osd":6,"last update":"Fri Mar 29 04:52:49 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.60899999999999999,"15min":0.625},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.82299999999999995,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.61899999999999999,"15min":0.63500000000000001},"min":{"1min":0.38500000000000001,"5min":0.375,"15min":0.375},"max":{"1min":0.73599999999999999,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.59699999999999998}]},{"osd":7,"last update":"Fri Mar 29 04:52:05 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.57299999999999995,"15min":0.57099999999999995},"min":{"1min":0.435,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.63600000000000001,"15min":0.64000000000000001},"min":{"1min":0.45100000000000001,"5min":0.44,"15min":0.44},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.51400000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149753,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":729088,"data_stored":316559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:52:32 2024","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.378,"15min":0.40400000000000003},"min":{"1min":0.24299999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.47999999999999998,"5min":0.53600000000000003,"15min":0.53600000000000003},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.40799999999999997,"15min":0.42899999999999999},"min":{"1min":0.253,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.56799999999999995,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.29699999999999999}]},{"osd":1,"last update":"Fri Mar 29 04:52:32 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.38600000000000001,"15min":0.40200000000000002},"min":{"1min":0.29899999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.48099999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.371,"5min":0.42999999999999999,"15min":0.45200000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.53900000000000003,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.442}]},{"osd":2,"last update":"Fri Mar 29 04:52:32 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.439,"15min":0.45700000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.48599999999999999,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.374,"5min":0.433,"15min":0.47899999999999998},"min":{"1min":0.26000000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.505,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.34200000000000003}]},{"osd":4,"last update":"Fri Mar 29 04:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.60399999999999998,"15min":0.61799999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.746,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58499999999999996,"15min":0.58899999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.71399999999999997}]},{"osd":5,"last update":"Fri Mar 29 04:52:19 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60699999999999998,"15min":0.61699999999999999},"min":{"1min":0.42699999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.75600000000000001,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.60599999999999998,"15min":0.61599999999999999},"min":{"1min":0.44500000000000001,"5min":0.434,"15min":0.434},"max":{"1min":0.69699999999999995,"5min":1.042,"15min":1.042},"last":0.69299999999999995}]},{"osd":6,"last update":"Fri Mar 29 04:52:49 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.437,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79100000000000004,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60699999999999998,"15min":0.61299999999999999},"min":{"1min":0.46600000000000003,"5min":0.373,"15min":0.373},"max":{"1min":0.73599999999999999,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.53700000000000003}]},{"osd":7,"last update":"Fri Mar 29 04:52:08 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.46200000000000002,"5min":0.436,"15min":0.436},"max":{"1min":0.77200000000000002,"5min":1.002,"15min":1.002},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.621,"5min":0.60099999999999998,"15min":0.59799999999999998},"min":{"1min":0.496,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.83199999999999996,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.57299999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986227,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298084,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447068,"statfs":{"total":95995035648,"available":95689797632,"internally_reserved":0,"allocated":741376,"data_stored":317637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:51:54 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.54000000000000004,"15min":0.54400000000000004},"min":{"1min":0.379,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.55400000000000005,"15min":0.56599999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.70199999999999996,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.69799999999999995}]},{"osd":1,"last update":"Fri Mar 29 04:51:54 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.53300000000000003,"15min":0.53600000000000003},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.73799999999999999,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.54400000000000004,"15min":0.55000000000000004},"min":{"1min":0.39800000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.77200000000000002,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.56799999999999995}]},{"osd":2,"last update":"Fri Mar 29 04:51:54 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.54800000000000004,"15min":0.55800000000000005},"min":{"1min":0.36499999999999999,"5min":0.312,"15min":0.312},"max":{"1min":0.749,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.498},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.53800000000000003,"15min":0.54100000000000004},"min":{"1min":0.38800000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.80400000000000005,"5min":0.877,"15min":0.877},"last":0.73799999999999999}]},{"osd":3,"last update":"Fri Mar 29 04:51:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.55000000000000004,"15min":0.55300000000000005},"min":{"1min":0.379,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.73699999999999999,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.56499999999999995,"15min":0.57099999999999995},"min":{"1min":0.436,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.78300000000000003,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.52700000000000002}]},{"osd":5,"last update":"Fri Mar 29 04:52:13 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.436,"15min":0.42199999999999999},"min":{"1min":0.249,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.69699999999999995,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.49299999999999999,"15min":0.47799999999999998},"min":{"1min":0.33000000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.71699999999999997,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.65200000000000002}]},{"osd":6,"last update":"Fri Mar 29 04:52:39 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.47899999999999998,"15min":0.47599999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.65200000000000002,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.503,"5min":0.48199999999999998,"15min":0.47499999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.64900000000000002,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.623}]},{"osd":7,"last update":"Fri Mar 29 04:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.63100000000000001,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.53500000000000003,"15min":0.53300000000000003},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.72899999999999998,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.63800000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724590,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":729088,"data_stored":316786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.53100000000000003,"15min":0.54100000000000004},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.68100000000000005,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.53200000000000003,"15min":0.55000000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.66900000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.57599999999999996}]},{"osd":1,"last update":"Fri Mar 29 04:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.53500000000000003,"15min":0.54500000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.66500000000000004,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.54700000000000004,"15min":0.55300000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.64500000000000002,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.48399999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.57399999999999995,"15min":0.58199999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.74399999999999999,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.53900000000000003,"15min":0.54400000000000004},"min":{"1min":0.36399999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.66800000000000004,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.41099999999999998}]},{"osd":3,"last update":"Fri Mar 29 04:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.53400000000000003,"15min":0.54000000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.753,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.497},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.57199999999999995,"15min":0.58199999999999996},"min":{"1min":0.40799999999999997,"5min":0.37,"15min":0.37},"max":{"1min":0.71099999999999997,"5min":0.753,"15min":0.753},"last":0.433}]},{"osd":4,"last update":"Fri Mar 29 04:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.42499999999999999,"15min":0.39600000000000002},"min":{"1min":0.32300000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.70099999999999996,"5min":1.075,"15min":1.075},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.442,"5min":0.438,"15min":0.42099999999999999},"min":{"1min":0.30099999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.59399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.44700000000000001}]},{"osd":6,"last update":"Fri Mar 29 04:52:43 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.47699999999999998,"15min":0.46100000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.63200000000000001,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.48399999999999999,"15min":0.45200000000000001},"min":{"1min":0.373,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.47099999999999997}]},{"osd":7,"last update":"Fri Mar 29 04:52:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.55100000000000005,"15min":0.54700000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.73099999999999998,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.55400000000000005,"15min":0.55200000000000005},"min":{"1min":0.40400000000000003,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78900000000000003,"5min":1.115,"15min":1.115},"last":0.54800000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T04:52:51.200 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T04:52:51.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:50 smithi187 ceph-mon[33595]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:51.455 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:52.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:51 smithi181 ceph-mon[29765]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:52:52.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:51 smithi181 ceph-mon[41183]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:52:52.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:51 smithi187 ceph-mon[33595]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:52:53.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:52 smithi181 ceph-mon[29765]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:53.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:52 smithi181 ceph-mon[41183]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:53.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:52 smithi187 ceph-mon[33595]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:53.388 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:52:53.390 INFO:teuthology.orchestra.run.smithi181.stderr:dumped all 2024-03-29T04:52:54.057 INFO:teuthology.orchestra.run.smithi181.stdout:{"pg_ready":true,"pg_map":{"version":83,"stamp":"2024-03-29T04:52:51.613101+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386320,"kb_used_data":7460,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574896,"statfs":{"total":767960285184,"available":765516693504,"internally_reserved":0,"allocated":7639040,"data_stored":4307046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002691"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689128+0000","last_change":"2024-03-29T04:51:50.905064+0000","last_active":"2024-03-29T04:52:33.689128+0000","last_peered":"2024-03-29T04:52:33.689128+0000","last_clean":"2024-03-29T04:52:33.689128+0000","last_became_active":"2024-03-29T04:51:50.904783+0000","last_became_peered":"2024-03-29T04:51:50.904783+0000","last_unstale":"2024-03-29T04:52:33.689128+0000","last_undegraded":"2024-03-29T04:52:33.689128+0000","last_fullsized":"2024-03-29T04:52:33.689128+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:15:06.303234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.691667+0000","last_change":"2024-03-29T04:51:50.887615+0000","last_active":"2024-03-29T04:52:33.691667+0000","last_peered":"2024-03-29T04:52:33.691667+0000","last_clean":"2024-03-29T04:52:33.691667+0000","last_became_active":"2024-03-29T04:51:50.887273+0000","last_became_peered":"2024-03-29T04:51:50.887273+0000","last_unstale":"2024-03-29T04:52:33.691667+0000","last_undegraded":"2024-03-29T04:52:33.691667+0000","last_fullsized":"2024-03-29T04:52:33.691667+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:24:36.868812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686457+0000","last_change":"2024-03-29T04:51:51.914368+0000","last_active":"2024-03-29T04:52:33.686457+0000","last_peered":"2024-03-29T04:52:33.686457+0000","last_clean":"2024-03-29T04:52:33.686457+0000","last_became_active":"2024-03-29T04:51:51.914060+0000","last_became_peered":"2024-03-29T04:51:51.914060+0000","last_unstale":"2024-03-29T04:52:33.686457+0000","last_undegraded":"2024-03-29T04:52:33.686457+0000","last_fullsized":"2024-03-29T04:52:33.686457+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:08:02.210421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686908+0000","last_change":"2024-03-29T04:51:50.891259+0000","last_active":"2024-03-29T04:52:33.686908+0000","last_peered":"2024-03-29T04:52:33.686908+0000","last_clean":"2024-03-29T04:52:33.686908+0000","last_became_active":"2024-03-29T04:51:50.890952+0000","last_became_peered":"2024-03-29T04:51:50.890952+0000","last_unstale":"2024-03-29T04:52:33.686908+0000","last_undegraded":"2024-03-29T04:52:33.686908+0000","last_fullsized":"2024-03-29T04:52:33.686908+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:01:46.663624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689217+0000","last_change":"2024-03-29T04:51:51.345846+0000","last_active":"2024-03-29T04:52:33.689217+0000","last_peered":"2024-03-29T04:52:33.689217+0000","last_clean":"2024-03-29T04:52:33.689217+0000","last_became_active":"2024-03-29T04:51:51.345551+0000","last_became_peered":"2024-03-29T04:51:51.345551+0000","last_unstale":"2024-03-29T04:52:33.689217+0000","last_undegraded":"2024-03-29T04:52:33.689217+0000","last_fullsized":"2024-03-29T04:52:33.689217+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:56:13.597807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690090+0000","last_change":"2024-03-29T04:51:50.895069+0000","last_active":"2024-03-29T04:52:33.690090+0000","last_peered":"2024-03-29T04:52:33.690090+0000","last_clean":"2024-03-29T04:52:33.690090+0000","last_became_active":"2024-03-29T04:51:50.894761+0000","last_became_peered":"2024-03-29T04:51:50.894761+0000","last_unstale":"2024-03-29T04:52:33.690090+0000","last_undegraded":"2024-03-29T04:52:33.690090+0000","last_fullsized":"2024-03-29T04:52:33.690090+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:52:53.436984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689596+0000","last_change":"2024-03-29T04:51:51.345898+0000","last_active":"2024-03-29T04:52:33.689596+0000","last_peered":"2024-03-29T04:52:33.689596+0000","last_clean":"2024-03-29T04:52:33.689596+0000","last_became_active":"2024-03-29T04:51:51.345579+0000","last_became_peered":"2024-03-29T04:51:51.345579+0000","last_unstale":"2024-03-29T04:52:33.689596+0000","last_undegraded":"2024-03-29T04:52:33.689596+0000","last_fullsized":"2024-03-29T04:52:33.689596+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:07:35.976888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689960+0000","last_change":"2024-03-29T04:51:50.892033+0000","last_active":"2024-03-29T04:52:33.689960+0000","last_peered":"2024-03-29T04:52:33.689960+0000","last_clean":"2024-03-29T04:52:33.689960+0000","last_became_active":"2024-03-29T04:51:50.890155+0000","last_became_peered":"2024-03-29T04:51:50.890155+0000","last_unstale":"2024-03-29T04:52:33.689960+0000","last_undegraded":"2024-03-29T04:52:33.689960+0000","last_fullsized":"2024-03-29T04:52:33.689960+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:58:23.398616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689094+0000","last_change":"2024-03-29T04:51:51.915466+0000","last_active":"2024-03-29T04:52:33.689094+0000","last_peered":"2024-03-29T04:52:33.689094+0000","last_clean":"2024-03-29T04:52:33.689094+0000","last_became_active":"2024-03-29T04:51:51.915121+0000","last_became_peered":"2024-03-29T04:51:51.915121+0000","last_unstale":"2024-03-29T04:52:33.689094+0000","last_undegraded":"2024-03-29T04:52:33.689094+0000","last_fullsized":"2024-03-29T04:52:33.689094+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:00:34.141526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.691440+0000","last_change":"2024-03-29T04:51:50.893229+0000","last_active":"2024-03-29T04:52:33.691440+0000","last_peered":"2024-03-29T04:52:33.691440+0000","last_clean":"2024-03-29T04:52:33.691440+0000","last_became_active":"2024-03-29T04:51:50.891671+0000","last_became_peered":"2024-03-29T04:51:50.891671+0000","last_unstale":"2024-03-29T04:52:33.691440+0000","last_undegraded":"2024-03-29T04:52:33.691440+0000","last_fullsized":"2024-03-29T04:52:33.691440+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:59:18.841471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.688976+0000","last_change":"2024-03-29T04:51:50.904139+0000","last_active":"2024-03-29T04:52:33.688976+0000","last_peered":"2024-03-29T04:52:33.688976+0000","last_clean":"2024-03-29T04:52:33.688976+0000","last_became_active":"2024-03-29T04:51:50.903780+0000","last_became_peered":"2024-03-29T04:51:50.903780+0000","last_unstale":"2024-03-29T04:52:33.688976+0000","last_undegraded":"2024-03-29T04:52:33.688976+0000","last_fullsized":"2024-03-29T04:52:33.688976+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:31:58.484667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.692078+0000","last_change":"2024-03-29T04:51:50.891247+0000","last_active":"2024-03-29T04:52:33.692078+0000","last_peered":"2024-03-29T04:52:33.692078+0000","last_clean":"2024-03-29T04:52:33.692078+0000","last_became_active":"2024-03-29T04:51:50.890257+0000","last_became_peered":"2024-03-29T04:51:50.890257+0000","last_unstale":"2024-03-29T04:52:33.692078+0000","last_undegraded":"2024-03-29T04:52:33.692078+0000","last_fullsized":"2024-03-29T04:52:33.692078+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:30:04.320976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913110+0000","last_change":"2024-03-29T04:50:39.046369+0000","last_active":"2024-03-29T04:52:33.913110+0000","last_peered":"2024-03-29T04:52:33.913110+0000","last_clean":"2024-03-29T04:52:33.913110+0000","last_became_active":"2024-03-29T04:50:39.046086+0000","last_became_peered":"2024-03-29T04:50:39.046086+0000","last_unstale":"2024-03-29T04:52:33.913110+0000","last_undegraded":"2024-03-29T04:52:33.913110+0000","last_fullsized":"2024-03-29T04:52:33.913110+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:26:33.515943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686221+0000","last_change":"2024-03-29T04:51:52.387075+0000","last_active":"2024-03-29T04:52:33.686221+0000","last_peered":"2024-03-29T04:52:33.686221+0000","last_clean":"2024-03-29T04:52:33.686221+0000","last_became_active":"2024-03-29T04:51:52.386763+0000","last_became_peered":"2024-03-29T04:51:52.386763+0000","last_unstale":"2024-03-29T04:52:33.686221+0000","last_undegraded":"2024-03-29T04:52:33.686221+0000","last_fullsized":"2024-03-29T04:52:33.686221+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:31:09.993264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690167+0000","last_change":"2024-03-29T04:50:39.220266+0000","last_active":"2024-03-29T04:52:33.690167+0000","last_peered":"2024-03-29T04:52:33.690167+0000","last_clean":"2024-03-29T04:52:33.690167+0000","last_became_active":"2024-03-29T04:50:39.219919+0000","last_became_peered":"2024-03-29T04:50:39.219919+0000","last_unstale":"2024-03-29T04:52:33.690167+0000","last_undegraded":"2024-03-29T04:52:33.690167+0000","last_fullsized":"2024-03-29T04:52:33.690167+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:16:10.835704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.687043+0000","last_change":"2024-03-29T04:51:50.892648+0000","last_active":"2024-03-29T04:52:33.687043+0000","last_peered":"2024-03-29T04:52:33.687043+0000","last_clean":"2024-03-29T04:52:33.687043+0000","last_became_active":"2024-03-29T04:51:50.892381+0000","last_became_peered":"2024-03-29T04:51:50.892381+0000","last_unstale":"2024-03-29T04:52:33.687043+0000","last_undegraded":"2024-03-29T04:52:33.687043+0000","last_fullsized":"2024-03-29T04:52:33.687043+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:25:06.918009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.691341+0000","last_change":"2024-03-29T04:51:52.386981+0000","last_active":"2024-03-29T04:52:33.691341+0000","last_peered":"2024-03-29T04:52:33.691341+0000","last_clean":"2024-03-29T04:52:33.691341+0000","last_became_active":"2024-03-29T04:51:52.386716+0000","last_became_peered":"2024-03-29T04:51:52.386716+0000","last_unstale":"2024-03-29T04:52:33.691341+0000","last_undegraded":"2024-03-29T04:52:33.691341+0000","last_fullsized":"2024-03-29T04:52:33.691341+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:09:33.409779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690145+0000","last_change":"2024-03-29T04:51:50.889224+0000","last_active":"2024-03-29T04:52:33.690145+0000","last_peered":"2024-03-29T04:52:33.690145+0000","last_clean":"2024-03-29T04:52:33.690145+0000","last_became_active":"2024-03-29T04:51:50.888933+0000","last_became_peered":"2024-03-29T04:51:50.888933+0000","last_unstale":"2024-03-29T04:52:33.690145+0000","last_undegraded":"2024-03-29T04:52:33.690145+0000","last_fullsized":"2024-03-29T04:52:33.690145+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:41:08.015792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913496+0000","last_change":"2024-03-29T04:50:39.219980+0000","last_active":"2024-03-29T04:52:33.913496+0000","last_peered":"2024-03-29T04:52:33.913496+0000","last_clean":"2024-03-29T04:52:33.913496+0000","last_became_active":"2024-03-29T04:50:39.219627+0000","last_became_peered":"2024-03-29T04:50:39.219627+0000","last_unstale":"2024-03-29T04:52:33.913496+0000","last_undegraded":"2024-03-29T04:52:33.913496+0000","last_fullsized":"2024-03-29T04:52:33.913496+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:56:32.992193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-29T04:52:34.965896+0000","last_change":"2024-03-29T04:52:34.965896+0000","last_active":"2024-03-29T04:52:34.965896+0000","last_peered":"2024-03-29T04:52:34.965896+0000","last_clean":"2024-03-29T04:52:34.965896+0000","last_became_active":"2024-03-29T04:52:34.965513+0000","last_became_peered":"2024-03-29T04:52:34.965513+0000","last_unstale":"2024-03-29T04:52:34.965896+0000","last_undegraded":"2024-03-29T04:52:34.965896+0000","last_fullsized":"2024-03-29T04:52:34.965896+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:09:46.323541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.691301+0000","last_change":"2024-03-29T04:51:50.890387+0000","last_active":"2024-03-29T04:52:33.691301+0000","last_peered":"2024-03-29T04:52:33.691301+0000","last_clean":"2024-03-29T04:52:33.691301+0000","last_became_active":"2024-03-29T04:51:50.890015+0000","last_became_peered":"2024-03-29T04:51:50.890015+0000","last_unstale":"2024-03-29T04:52:33.691301+0000","last_undegraded":"2024-03-29T04:52:33.691301+0000","last_fullsized":"2024-03-29T04:52:33.691301+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:40:09.378351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913157+0000","last_change":"2024-03-29T04:50:39.149486+0000","last_active":"2024-03-29T04:52:33.913157+0000","last_peered":"2024-03-29T04:52:33.913157+0000","last_clean":"2024-03-29T04:52:33.913157+0000","last_became_active":"2024-03-29T04:50:39.149236+0000","last_became_peered":"2024-03-29T04:50:39.149236+0000","last_unstale":"2024-03-29T04:52:33.913157+0000","last_undegraded":"2024-03-29T04:52:33.913157+0000","last_fullsized":"2024-03-29T04:52:33.913157+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:14:50.050422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913220+0000","last_change":"2024-03-29T04:51:51.765782+0000","last_active":"2024-03-29T04:52:33.913220+0000","last_peered":"2024-03-29T04:52:33.913220+0000","last_clean":"2024-03-29T04:52:33.913220+0000","last_became_active":"2024-03-29T04:51:51.765491+0000","last_became_peered":"2024-03-29T04:51:51.765491+0000","last_unstale":"2024-03-29T04:52:33.913220+0000","last_undegraded":"2024-03-29T04:52:33.913220+0000","last_fullsized":"2024-03-29T04:52:33.913220+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:18:02.508416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686587+0000","last_change":"2024-03-29T04:51:50.892178+0000","last_active":"2024-03-29T04:52:33.686587+0000","last_peered":"2024-03-29T04:52:33.686587+0000","last_clean":"2024-03-29T04:52:33.686587+0000","last_became_active":"2024-03-29T04:51:50.891686+0000","last_became_peered":"2024-03-29T04:51:50.891686+0000","last_unstale":"2024-03-29T04:52:33.686587+0000","last_undegraded":"2024-03-29T04:52:33.686587+0000","last_fullsized":"2024-03-29T04:52:33.686587+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:24:14.883625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":278,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913049+0000","last_change":"2024-03-29T04:50:05.432965+0000","last_active":"2024-03-29T04:52:33.913049+0000","last_peered":"2024-03-29T04:52:33.913049+0000","last_clean":"2024-03-29T04:52:33.913049+0000","last_became_active":"2024-03-29T04:50:05.432530+0000","last_became_peered":"2024-03-29T04:50:05.432530+0000","last_unstale":"2024-03-29T04:52:33.913049+0000","last_undegraded":"2024-03-29T04:52:33.913049+0000","last_fullsized":"2024-03-29T04:52:33.913049+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:48:01.601528+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:48:01.601528+0000","last_clean_scrub_stamp":"2024-03-29T04:48:01.601528+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:56:20.936763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689817+0000","last_change":"2024-03-29T04:50:39.045035+0000","last_active":"2024-03-29T04:52:33.689817+0000","last_peered":"2024-03-29T04:52:33.689817+0000","last_clean":"2024-03-29T04:52:33.689817+0000","last_became_active":"2024-03-29T04:50:39.044724+0000","last_became_peered":"2024-03-29T04:50:39.044724+0000","last_unstale":"2024-03-29T04:52:33.689817+0000","last_undegraded":"2024-03-29T04:52:33.689817+0000","last_fullsized":"2024-03-29T04:52:33.689817+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:44:08.632467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689771+0000","last_change":"2024-03-29T04:51:51.281211+0000","last_active":"2024-03-29T04:52:33.689771+0000","last_peered":"2024-03-29T04:52:33.689771+0000","last_clean":"2024-03-29T04:52:33.689771+0000","last_became_active":"2024-03-29T04:51:51.280893+0000","last_became_peered":"2024-03-29T04:51:51.280893+0000","last_unstale":"2024-03-29T04:52:33.689771+0000","last_undegraded":"2024-03-29T04:52:33.689771+0000","last_fullsized":"2024-03-29T04:52:33.689771+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:37:24.853594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.688835+0000","last_change":"2024-03-29T04:51:50.892366+0000","last_active":"2024-03-29T04:52:33.688835+0000","last_peered":"2024-03-29T04:52:33.688835+0000","last_clean":"2024-03-29T04:52:33.688835+0000","last_became_active":"2024-03-29T04:51:50.892101+0000","last_became_peered":"2024-03-29T04:51:50.892101+0000","last_unstale":"2024-03-29T04:52:33.688835+0000","last_undegraded":"2024-03-29T04:52:33.688835+0000","last_fullsized":"2024-03-29T04:52:33.688835+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:52:18.835709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689033+0000","last_change":"2024-03-29T04:50:39.246218+0000","last_active":"2024-03-29T04:52:33.689033+0000","last_peered":"2024-03-29T04:52:33.689033+0000","last_clean":"2024-03-29T04:52:33.689033+0000","last_became_active":"2024-03-29T04:50:39.245880+0000","last_became_peered":"2024-03-29T04:50:39.245880+0000","last_unstale":"2024-03-29T04:52:33.689033+0000","last_undegraded":"2024-03-29T04:52:33.689033+0000","last_fullsized":"2024-03-29T04:52:33.689033+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:17:07.467340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-29T04:52:34.965706+0000","last_change":"2024-03-29T04:52:34.965706+0000","last_active":"2024-03-29T04:52:34.965706+0000","last_peered":"2024-03-29T04:52:34.965706+0000","last_clean":"2024-03-29T04:52:34.965706+0000","last_became_active":"2024-03-29T04:52:34.965452+0000","last_became_peered":"2024-03-29T04:52:34.965452+0000","last_unstale":"2024-03-29T04:52:34.965706+0000","last_undegraded":"2024-03-29T04:52:34.965706+0000","last_fullsized":"2024-03-29T04:52:34.965706+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:35:28.368820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689231+0000","last_change":"2024-03-29T04:51:50.891485+0000","last_active":"2024-03-29T04:52:33.689231+0000","last_peered":"2024-03-29T04:52:33.689231+0000","last_clean":"2024-03-29T04:52:33.689231+0000","last_became_active":"2024-03-29T04:51:50.891222+0000","last_became_peered":"2024-03-29T04:51:50.891222+0000","last_unstale":"2024-03-29T04:52:33.689231+0000","last_undegraded":"2024-03-29T04:52:33.689231+0000","last_fullsized":"2024-03-29T04:52:33.689231+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:51:11.170455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689479+0000","last_change":"2024-03-29T04:50:39.044530+0000","last_active":"2024-03-29T04:52:33.689479+0000","last_peered":"2024-03-29T04:52:33.689479+0000","last_clean":"2024-03-29T04:52:33.689479+0000","last_became_active":"2024-03-29T04:50:39.044136+0000","last_became_peered":"2024-03-29T04:50:39.044136+0000","last_unstale":"2024-03-29T04:52:33.689479+0000","last_undegraded":"2024-03-29T04:52:33.689479+0000","last_fullsized":"2024-03-29T04:52:33.689479+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:32:12.577651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.688856+0000","last_change":"2024-03-29T04:51:50.904279+0000","last_active":"2024-03-29T04:52:33.688856+0000","last_peered":"2024-03-29T04:52:33.688856+0000","last_clean":"2024-03-29T04:52:33.688856+0000","last_became_active":"2024-03-29T04:51:50.904049+0000","last_became_peered":"2024-03-29T04:51:50.904049+0000","last_unstale":"2024-03-29T04:52:33.688856+0000","last_undegraded":"2024-03-29T04:52:33.688856+0000","last_fullsized":"2024-03-29T04:52:33.688856+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:44:42.492894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689998+0000","last_change":"2024-03-29T04:51:50.887938+0000","last_active":"2024-03-29T04:52:33.689998+0000","last_peered":"2024-03-29T04:52:33.689998+0000","last_clean":"2024-03-29T04:52:33.689998+0000","last_became_active":"2024-03-29T04:51:50.887501+0000","last_became_peered":"2024-03-29T04:51:50.887501+0000","last_unstale":"2024-03-29T04:52:33.689998+0000","last_undegraded":"2024-03-29T04:52:33.689998+0000","last_fullsized":"2024-03-29T04:52:33.689998+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:27:22.602641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690051+0000","last_change":"2024-03-29T04:50:39.149460+0000","last_active":"2024-03-29T04:52:33.690051+0000","last_peered":"2024-03-29T04:52:33.690051+0000","last_clean":"2024-03-29T04:52:33.690051+0000","last_became_active":"2024-03-29T04:50:39.149152+0000","last_became_peered":"2024-03-29T04:50:39.149152+0000","last_unstale":"2024-03-29T04:52:33.690051+0000","last_undegraded":"2024-03-29T04:52:33.690051+0000","last_fullsized":"2024-03-29T04:52:33.690051+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:05:37.697946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689700+0000","last_change":"2024-03-29T04:51:51.346261+0000","last_active":"2024-03-29T04:52:33.689700+0000","last_peered":"2024-03-29T04:52:33.689700+0000","last_clean":"2024-03-29T04:52:33.689700+0000","last_became_active":"2024-03-29T04:51:51.345637+0000","last_became_peered":"2024-03-29T04:51:51.345637+0000","last_unstale":"2024-03-29T04:52:33.689700+0000","last_undegraded":"2024-03-29T04:52:33.689700+0000","last_fullsized":"2024-03-29T04:52:33.689700+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:50:03.025362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686221+0000","last_change":"2024-03-29T04:51:50.890374+0000","last_active":"2024-03-29T04:52:33.686221+0000","last_peered":"2024-03-29T04:52:33.686221+0000","last_clean":"2024-03-29T04:52:33.686221+0000","last_became_active":"2024-03-29T04:51:50.890087+0000","last_became_peered":"2024-03-29T04:51:50.890087+0000","last_unstale":"2024-03-29T04:52:33.686221+0000","last_undegraded":"2024-03-29T04:52:33.686221+0000","last_fullsized":"2024-03-29T04:52:33.686221+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:58:26.599714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690156+0000","last_change":"2024-03-29T04:51:50.905623+0000","last_active":"2024-03-29T04:52:33.690156+0000","last_peered":"2024-03-29T04:52:33.690156+0000","last_clean":"2024-03-29T04:52:33.690156+0000","last_became_active":"2024-03-29T04:51:50.905346+0000","last_became_peered":"2024-03-29T04:51:50.905346+0000","last_unstale":"2024-03-29T04:52:33.690156+0000","last_undegraded":"2024-03-29T04:52:33.690156+0000","last_fullsized":"2024-03-29T04:52:33.690156+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:05:20.465451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689027+0000","last_change":"2024-03-29T04:51:50.892490+0000","last_active":"2024-03-29T04:52:33.689027+0000","last_peered":"2024-03-29T04:52:33.689027+0000","last_clean":"2024-03-29T04:52:33.689027+0000","last_became_active":"2024-03-29T04:51:50.892051+0000","last_became_peered":"2024-03-29T04:51:50.892051+0000","last_unstale":"2024-03-29T04:52:33.689027+0000","last_undegraded":"2024-03-29T04:52:33.689027+0000","last_fullsized":"2024-03-29T04:52:33.689027+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:44:19.582886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690536+0000","last_change":"2024-03-29T04:51:51.234266+0000","last_active":"2024-03-29T04:52:33.690536+0000","last_peered":"2024-03-29T04:52:33.690536+0000","last_clean":"2024-03-29T04:52:33.690536+0000","last_became_active":"2024-03-29T04:51:51.233943+0000","last_became_peered":"2024-03-29T04:51:51.233943+0000","last_unstale":"2024-03-29T04:52:33.690536+0000","last_undegraded":"2024-03-29T04:52:33.690536+0000","last_fullsized":"2024-03-29T04:52:33.690536+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:30:49.735372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913057+0000","last_change":"2024-03-29T04:51:50.897624+0000","last_active":"2024-03-29T04:52:33.913057+0000","last_peered":"2024-03-29T04:52:33.913057+0000","last_clean":"2024-03-29T04:52:33.913057+0000","last_became_active":"2024-03-29T04:51:50.897348+0000","last_became_peered":"2024-03-29T04:51:50.897348+0000","last_unstale":"2024-03-29T04:52:33.913057+0000","last_undegraded":"2024-03-29T04:52:33.913057+0000","last_fullsized":"2024-03-29T04:52:33.913057+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:16:56.513567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690100+0000","last_change":"2024-03-29T04:51:51.346008+0000","last_active":"2024-03-29T04:52:33.690100+0000","last_peered":"2024-03-29T04:52:33.690100+0000","last_clean":"2024-03-29T04:52:33.690100+0000","last_became_active":"2024-03-29T04:51:51.345554+0000","last_became_peered":"2024-03-29T04:51:51.345554+0000","last_unstale":"2024-03-29T04:52:33.690100+0000","last_undegraded":"2024-03-29T04:52:33.690100+0000","last_fullsized":"2024-03-29T04:52:33.690100+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:04:57.630878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686642+0000","last_change":"2024-03-29T04:51:50.894562+0000","last_active":"2024-03-29T04:52:33.686642+0000","last_peered":"2024-03-29T04:52:33.686642+0000","last_clean":"2024-03-29T04:52:33.686642+0000","last_became_active":"2024-03-29T04:51:50.894244+0000","last_became_peered":"2024-03-29T04:51:50.894244+0000","last_unstale":"2024-03-29T04:52:33.686642+0000","last_undegraded":"2024-03-29T04:52:33.686642+0000","last_fullsized":"2024-03-29T04:52:33.686642+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:41:22.958621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.691530+0000","last_change":"2024-03-29T04:51:52.096173+0000","last_active":"2024-03-29T04:52:33.691530+0000","last_peered":"2024-03-29T04:52:33.691530+0000","last_clean":"2024-03-29T04:52:33.691530+0000","last_became_active":"2024-03-29T04:51:52.095857+0000","last_became_peered":"2024-03-29T04:51:52.095857+0000","last_unstale":"2024-03-29T04:52:33.691530+0000","last_undegraded":"2024-03-29T04:52:33.691530+0000","last_fullsized":"2024-03-29T04:52:33.691530+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:47:28.087749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689849+0000","last_change":"2024-03-29T04:51:50.889041+0000","last_active":"2024-03-29T04:52:33.689849+0000","last_peered":"2024-03-29T04:52:33.689849+0000","last_clean":"2024-03-29T04:52:33.689849+0000","last_became_active":"2024-03-29T04:51:50.888733+0000","last_became_peered":"2024-03-29T04:51:50.888733+0000","last_unstale":"2024-03-29T04:52:33.689849+0000","last_undegraded":"2024-03-29T04:52:33.689849+0000","last_fullsized":"2024-03-29T04:52:33.689849+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:06:45.856873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.912879+0000","last_change":"2024-03-29T04:51:51.764934+0000","last_active":"2024-03-29T04:52:33.912879+0000","last_peered":"2024-03-29T04:52:33.912879+0000","last_clean":"2024-03-29T04:52:33.912879+0000","last_became_active":"2024-03-29T04:51:51.764617+0000","last_became_peered":"2024-03-29T04:51:51.764617+0000","last_unstale":"2024-03-29T04:52:33.912879+0000","last_undegraded":"2024-03-29T04:52:33.912879+0000","last_fullsized":"2024-03-29T04:52:33.912879+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:33:50.709751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913643+0000","last_change":"2024-03-29T04:51:50.896476+0000","last_active":"2024-03-29T04:52:33.913643+0000","last_peered":"2024-03-29T04:52:33.913643+0000","last_clean":"2024-03-29T04:52:33.913643+0000","last_became_active":"2024-03-29T04:51:50.895258+0000","last_became_peered":"2024-03-29T04:51:50.895258+0000","last_unstale":"2024-03-29T04:52:33.913643+0000","last_undegraded":"2024-03-29T04:52:33.913643+0000","last_fullsized":"2024-03-29T04:52:33.913643+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:24:42.583134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.688855+0000","last_change":"2024-03-29T04:51:51.345825+0000","last_active":"2024-03-29T04:52:33.688855+0000","last_peered":"2024-03-29T04:52:33.688855+0000","last_clean":"2024-03-29T04:52:33.688855+0000","last_became_active":"2024-03-29T04:51:51.345484+0000","last_became_peered":"2024-03-29T04:51:51.345484+0000","last_unstale":"2024-03-29T04:52:33.688855+0000","last_undegraded":"2024-03-29T04:52:33.688855+0000","last_fullsized":"2024-03-29T04:52:33.688855+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:12:47.292042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913636+0000","last_change":"2024-03-29T04:51:50.890021+0000","last_active":"2024-03-29T04:52:33.913636+0000","last_peered":"2024-03-29T04:52:33.913636+0000","last_clean":"2024-03-29T04:52:33.913636+0000","last_became_active":"2024-03-29T04:51:50.889698+0000","last_became_peered":"2024-03-29T04:51:50.889698+0000","last_unstale":"2024-03-29T04:52:33.913636+0000","last_undegraded":"2024-03-29T04:52:33.913636+0000","last_fullsized":"2024-03-29T04:52:33.913636+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:49:55.193916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689466+0000","last_change":"2024-03-29T04:51:51.764872+0000","last_active":"2024-03-29T04:52:33.689466+0000","last_peered":"2024-03-29T04:52:33.689466+0000","last_clean":"2024-03-29T04:52:33.689466+0000","last_became_active":"2024-03-29T04:51:51.764554+0000","last_became_peered":"2024-03-29T04:51:51.764554+0000","last_unstale":"2024-03-29T04:52:33.689466+0000","last_undegraded":"2024-03-29T04:52:33.689466+0000","last_fullsized":"2024-03-29T04:52:33.689466+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:09:07.383326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913717+0000","last_change":"2024-03-29T04:51:50.895042+0000","last_active":"2024-03-29T04:52:33.913717+0000","last_peered":"2024-03-29T04:52:33.913717+0000","last_clean":"2024-03-29T04:52:33.913717+0000","last_became_active":"2024-03-29T04:51:50.894731+0000","last_became_peered":"2024-03-29T04:51:50.894731+0000","last_unstale":"2024-03-29T04:52:33.913717+0000","last_undegraded":"2024-03-29T04:52:33.913717+0000","last_fullsized":"2024-03-29T04:52:33.913717+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:30:15.164924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686937+0000","last_change":"2024-03-29T04:51:52.389279+0000","last_active":"2024-03-29T04:52:33.686937+0000","last_peered":"2024-03-29T04:52:33.686937+0000","last_clean":"2024-03-29T04:52:33.686937+0000","last_became_active":"2024-03-29T04:51:52.388996+0000","last_became_peered":"2024-03-29T04:51:52.388996+0000","last_unstale":"2024-03-29T04:52:33.686937+0000","last_undegraded":"2024-03-29T04:52:33.686937+0000","last_fullsized":"2024-03-29T04:52:33.686937+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:41:32.316840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913170+0000","last_change":"2024-03-29T04:51:50.892088+0000","last_active":"2024-03-29T04:52:33.913170+0000","last_peered":"2024-03-29T04:52:33.913170+0000","last_clean":"2024-03-29T04:52:33.913170+0000","last_became_active":"2024-03-29T04:51:50.891771+0000","last_became_peered":"2024-03-29T04:51:50.891771+0000","last_unstale":"2024-03-29T04:52:33.913170+0000","last_undegraded":"2024-03-29T04:52:33.913170+0000","last_fullsized":"2024-03-29T04:52:33.913170+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:29:38.982286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.691964+0000","last_change":"2024-03-29T04:51:52.388437+0000","last_active":"2024-03-29T04:52:33.691964+0000","last_peered":"2024-03-29T04:52:33.691964+0000","last_clean":"2024-03-29T04:52:33.691964+0000","last_became_active":"2024-03-29T04:51:52.388167+0000","last_became_peered":"2024-03-29T04:51:52.388167+0000","last_unstale":"2024-03-29T04:52:33.691964+0000","last_undegraded":"2024-03-29T04:52:33.691964+0000","last_fullsized":"2024-03-29T04:52:33.691964+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:09:14.684669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.912865+0000","last_change":"2024-03-29T04:51:50.897482+0000","last_active":"2024-03-29T04:52:33.912865+0000","last_peered":"2024-03-29T04:52:33.912865+0000","last_clean":"2024-03-29T04:52:33.912865+0000","last_became_active":"2024-03-29T04:51:50.896991+0000","last_became_peered":"2024-03-29T04:51:50.896991+0000","last_unstale":"2024-03-29T04:52:33.912865+0000","last_undegraded":"2024-03-29T04:52:33.912865+0000","last_fullsized":"2024-03-29T04:52:33.912865+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:53:18.957886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689899+0000","last_change":"2024-03-29T04:51:51.766321+0000","last_active":"2024-03-29T04:52:33.689899+0000","last_peered":"2024-03-29T04:52:33.689899+0000","last_clean":"2024-03-29T04:52:33.689899+0000","last_became_active":"2024-03-29T04:51:51.765964+0000","last_became_peered":"2024-03-29T04:51:51.765964+0000","last_unstale":"2024-03-29T04:52:33.689899+0000","last_undegraded":"2024-03-29T04:52:33.689899+0000","last_fullsized":"2024-03-29T04:52:33.689899+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:16:59.703129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.691406+0000","last_change":"2024-03-29T04:51:50.889856+0000","last_active":"2024-03-29T04:52:33.691406+0000","last_peered":"2024-03-29T04:52:33.691406+0000","last_clean":"2024-03-29T04:52:33.691406+0000","last_became_active":"2024-03-29T04:51:50.889578+0000","last_became_peered":"2024-03-29T04:51:50.889578+0000","last_unstale":"2024-03-29T04:52:33.691406+0000","last_undegraded":"2024-03-29T04:52:33.691406+0000","last_fullsized":"2024-03-29T04:52:33.691406+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:39:31.797252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686773+0000","last_change":"2024-03-29T04:51:52.388513+0000","last_active":"2024-03-29T04:52:33.686773+0000","last_peered":"2024-03-29T04:52:33.686773+0000","last_clean":"2024-03-29T04:52:33.686773+0000","last_became_active":"2024-03-29T04:51:52.387739+0000","last_became_peered":"2024-03-29T04:51:52.387739+0000","last_unstale":"2024-03-29T04:52:33.686773+0000","last_undegraded":"2024-03-29T04:52:33.686773+0000","last_fullsized":"2024-03-29T04:52:33.686773+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:38:26.826741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913367+0000","last_change":"2024-03-29T04:51:50.896025+0000","last_active":"2024-03-29T04:52:33.913367+0000","last_peered":"2024-03-29T04:52:33.913367+0000","last_clean":"2024-03-29T04:52:33.913367+0000","last_became_active":"2024-03-29T04:51:50.895639+0000","last_became_peered":"2024-03-29T04:51:50.895639+0000","last_unstale":"2024-03-29T04:52:33.913367+0000","last_undegraded":"2024-03-29T04:52:33.913367+0000","last_fullsized":"2024-03-29T04:52:33.913367+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:42:09.132536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.689777+0000","last_change":"2024-03-29T04:51:50.907295+0000","last_active":"2024-03-29T04:52:33.689777+0000","last_peered":"2024-03-29T04:52:33.689777+0000","last_clean":"2024-03-29T04:52:33.689777+0000","last_became_active":"2024-03-29T04:51:50.906975+0000","last_became_peered":"2024-03-29T04:51:50.906975+0000","last_unstale":"2024-03-29T04:52:33.689777+0000","last_undegraded":"2024-03-29T04:52:33.689777+0000","last_fullsized":"2024-03-29T04:52:33.689777+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:47:28.195544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690080+0000","last_change":"2024-03-29T04:51:50.890463+0000","last_active":"2024-03-29T04:52:33.690080+0000","last_peered":"2024-03-29T04:52:33.690080+0000","last_clean":"2024-03-29T04:52:33.690080+0000","last_became_active":"2024-03-29T04:51:50.890171+0000","last_became_peered":"2024-03-29T04:51:50.890171+0000","last_unstale":"2024-03-29T04:52:33.690080+0000","last_undegraded":"2024-03-29T04:52:33.690080+0000","last_fullsized":"2024-03-29T04:52:33.690080+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:08:23.559107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.688767+0000","last_change":"2024-03-29T04:51:51.234068+0000","last_active":"2024-03-29T04:52:33.688767+0000","last_peered":"2024-03-29T04:52:33.688767+0000","last_clean":"2024-03-29T04:52:33.688767+0000","last_became_active":"2024-03-29T04:51:51.233793+0000","last_became_peered":"2024-03-29T04:51:51.233793+0000","last_unstale":"2024-03-29T04:52:33.688767+0000","last_undegraded":"2024-03-29T04:52:33.688767+0000","last_fullsized":"2024-03-29T04:52:33.688767+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:29:37.505596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.913490+0000","last_change":"2024-03-29T04:51:50.889057+0000","last_active":"2024-03-29T04:52:33.913490+0000","last_peered":"2024-03-29T04:52:33.913490+0000","last_clean":"2024-03-29T04:52:33.913490+0000","last_became_active":"2024-03-29T04:51:50.888695+0000","last_became_peered":"2024-03-29T04:51:50.888695+0000","last_unstale":"2024-03-29T04:52:33.913490+0000","last_undegraded":"2024-03-29T04:52:33.913490+0000","last_fullsized":"2024-03-29T04:52:33.913490+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:12:03.714559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686467+0000","last_change":"2024-03-29T04:51:52.388562+0000","last_active":"2024-03-29T04:52:33.686467+0000","last_peered":"2024-03-29T04:52:33.686467+0000","last_clean":"2024-03-29T04:52:33.686467+0000","last_became_active":"2024-03-29T04:51:52.388312+0000","last_became_peered":"2024-03-29T04:51:52.388312+0000","last_unstale":"2024-03-29T04:52:33.686467+0000","last_undegraded":"2024-03-29T04:52:33.686467+0000","last_fullsized":"2024-03-29T04:52:33.686467+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:26:24.263187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.690179+0000","last_change":"2024-03-29T04:51:50.891872+0000","last_active":"2024-03-29T04:52:33.690179+0000","last_peered":"2024-03-29T04:52:33.690179+0000","last_clean":"2024-03-29T04:52:33.690179+0000","last_became_active":"2024-03-29T04:51:50.891510+0000","last_became_peered":"2024-03-29T04:51:50.891510+0000","last_unstale":"2024-03-29T04:52:33.690179+0000","last_undegraded":"2024-03-29T04:52:33.690179+0000","last_fullsized":"2024-03-29T04:52:33.690179+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:18:15.904788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686318+0000","last_change":"2024-03-29T04:51:52.386593+0000","last_active":"2024-03-29T04:52:33.686318+0000","last_peered":"2024-03-29T04:52:33.686318+0000","last_clean":"2024-03-29T04:52:33.686318+0000","last_became_active":"2024-03-29T04:51:52.386289+0000","last_became_peered":"2024-03-29T04:51:52.386289+0000","last_unstale":"2024-03-29T04:52:33.686318+0000","last_undegraded":"2024-03-29T04:52:33.686318+0000","last_fullsized":"2024-03-29T04:52:33.686318+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:01:03.675008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.691661+0000","last_change":"2024-03-29T04:51:50.886309+0000","last_active":"2024-03-29T04:52:33.691661+0000","last_peered":"2024-03-29T04:52:33.691661+0000","last_clean":"2024-03-29T04:52:33.691661+0000","last_became_active":"2024-03-29T04:51:50.885946+0000","last_became_peered":"2024-03-29T04:51:50.885946+0000","last_unstale":"2024-03-29T04:52:33.691661+0000","last_undegraded":"2024-03-29T04:52:33.691661+0000","last_fullsized":"2024-03-29T04:52:33.691661+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:42:31.532903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.912995+0000","last_change":"2024-03-29T04:51:50.904596+0000","last_active":"2024-03-29T04:52:33.912995+0000","last_peered":"2024-03-29T04:52:33.912995+0000","last_clean":"2024-03-29T04:52:33.912995+0000","last_became_active":"2024-03-29T04:51:50.904303+0000","last_became_peered":"2024-03-29T04:51:50.904303+0000","last_unstale":"2024-03-29T04:52:33.912995+0000","last_undegraded":"2024-03-29T04:52:33.912995+0000","last_fullsized":"2024-03-29T04:52:33.912995+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:43:26.435518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.686426+0000","last_change":"2024-03-29T04:51:50.892768+0000","last_active":"2024-03-29T04:52:33.686426+0000","last_peered":"2024-03-29T04:52:33.686426+0000","last_clean":"2024-03-29T04:52:33.686426+0000","last_became_active":"2024-03-29T04:51:50.891998+0000","last_became_peered":"2024-03-29T04:51:50.891998+0000","last_unstale":"2024-03-29T04:52:33.686426+0000","last_undegraded":"2024-03-29T04:52:33.686426+0000","last_fullsized":"2024-03-29T04:52:33.686426+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:23:22.977219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.692061+0000","last_change":"2024-03-29T04:51:52.388697+0000","last_active":"2024-03-29T04:52:33.692061+0000","last_peered":"2024-03-29T04:52:33.692061+0000","last_clean":"2024-03-29T04:52:33.692061+0000","last_became_active":"2024-03-29T04:51:52.388436+0000","last_became_peered":"2024-03-29T04:51:52.388436+0000","last_unstale":"2024-03-29T04:52:33.692061+0000","last_undegraded":"2024-03-29T04:52:33.692061+0000","last_fullsized":"2024-03-29T04:52:33.692061+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:14:53.361219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.688747+0000","last_change":"2024-03-29T04:51:50.890473+0000","last_active":"2024-03-29T04:52:33.688747+0000","last_peered":"2024-03-29T04:52:33.688747+0000","last_clean":"2024-03-29T04:52:33.688747+0000","last_became_active":"2024-03-29T04:51:50.889848+0000","last_became_peered":"2024-03-29T04:51:50.889848+0000","last_unstale":"2024-03-29T04:52:33.688747+0000","last_undegraded":"2024-03-29T04:52:33.688747+0000","last_fullsized":"2024-03-29T04:52:33.688747+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:37:30.285886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-29T04:52:35.115723+0000","last_change":"2024-03-29T04:52:35.115723+0000","last_active":"2024-03-29T04:52:35.115723+0000","last_peered":"2024-03-29T04:52:35.115723+0000","last_clean":"2024-03-29T04:52:35.115723+0000","last_became_active":"2024-03-29T04:52:35.115424+0000","last_became_peered":"2024-03-29T04:52:35.115424+0000","last_unstale":"2024-03-29T04:52:35.115723+0000","last_undegraded":"2024-03-29T04:52:35.115723+0000","last_fullsized":"2024-03-29T04:52:35.115723+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:29:02.714701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:52:33.688974+0000","last_change":"2024-03-29T04:51:50.901011+0000","last_active":"2024-03-29T04:52:33.688974+0000","last_peered":"2024-03-29T04:52:33.688974+0000","last_clean":"2024-03-29T04:52:33.688974+0000","last_became_active":"2024-03-29T04:51:50.900150+0000","last_became_peered":"2024-03-29T04:51:50.900150+0000","last_unstale":"2024-03-29T04:52:33.688974+0000","last_undegraded":"2024-03-29T04:52:33.688974+0000","last_fullsized":"2024-03-29T04:52:33.688974+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:48:34.206192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201316,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298672,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446480,"statfs":{"total":95995035648,"available":95689195520,"internally_reserved":0,"allocated":1343488,"data_stored":908282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54600000000000004,"15min":0.54700000000000004},"min":{"1min":0.40600000000000003,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.65200000000000002,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.53900000000000003,"15min":0.54300000000000004},"min":{"1min":0.33200000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.71899999999999997,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.501}]},{"osd":1,"last update":"Fri Mar 29 04:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55900000000000005,"15min":0.56000000000000005},"min":{"1min":0.371,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.76000000000000001,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.375},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.438}]},{"osd":2,"last update":"Fri Mar 29 04:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.54500000000000004,"15min":0.54900000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.75,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.41499999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.82999999999999996,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.56699999999999995}]},{"osd":3,"last update":"Fri Mar 29 04:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.40799999999999997,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.69899999999999995,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.40899999999999997,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.80200000000000005,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.47599999999999998}]},{"osd":4,"last update":"Fri Mar 29 04:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.48999999999999999,"15min":0.496},"min":{"1min":0.36799999999999999,"5min":0.313,"15min":0.313},"max":{"1min":0.58699999999999997,"5min":0.879,"15min":0.879},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.502,"15min":0.502},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.58799999999999997,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.58299999999999996}]},{"osd":5,"last update":"Fri Mar 29 04:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51100000000000001,"15min":0.51000000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.78700000000000003,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.53800000000000003,"15min":0.54000000000000004},"min":{"1min":0.40799999999999997,"5min":0.37,"15min":0.37},"max":{"1min":0.625,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.55000000000000004}]},{"osd":6,"last update":"Fri Mar 29 04:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.71199999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.55000000000000004,"15min":0.54800000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.74299999999999999,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.623}]}]},{"osd":6,"up_from":47,"seq":201863462954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298652,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446500,"statfs":{"total":95995035648,"available":95689216000,"internally_reserved":0,"allocated":1327104,"data_stored":907871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52700000000000002,"15min":0.53000000000000003},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52500000000000002,"15min":0.53600000000000003},"min":{"1min":0.41899999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.66000000000000003,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.46200000000000002}]},{"osd":1,"last update":"Fri Mar 29 04:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54000000000000004,"15min":0.54300000000000004},"min":{"1min":0.35199999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.55100000000000005,"15min":0.56200000000000006},"min":{"1min":0.41499999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73499999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.626}]},{"osd":2,"last update":"Fri Mar 29 04:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.53700000000000003,"15min":0.54400000000000004},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.53300000000000003,"15min":0.53400000000000003},"min":{"1min":0.39600000000000002,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.40500000000000003}]},{"osd":3,"last update":"Fri Mar 29 04:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.55300000000000005,"15min":0.55600000000000005},"min":{"1min":0.40200000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54600000000000004,"15min":0.55400000000000005},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.69899999999999995,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.54800000000000004}]},{"osd":4,"last update":"Fri Mar 29 04:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.45500000000000002,"15min":0.44600000000000001},"min":{"1min":0.313,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.69199999999999995,"5min":0.746,"15min":0.746},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.45600000000000002,"15min":0.42999999999999999},"min":{"1min":0.33600000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.67500000000000004,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.51100000000000001}]},{"osd":5,"last update":"Fri Mar 29 04:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.47299999999999998,"15min":0.443},"min":{"1min":0.36799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.67400000000000004,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.501,"15min":0.47099999999999997},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.73099999999999998,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.64100000000000001}]},{"osd":7,"last update":"Fri Mar 29 04:52:06 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.51200000000000001,"15min":0.50800000000000001},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.54100000000000004,"15min":0.53500000000000003},"min":{"1min":0.40400000000000003,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.67500000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298056,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447096,"statfs":{"total":95995035648,"available":95689826304,"internally_reserved":0,"allocated":716800,"data_stored":315939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:51:45 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.35199999999999998,"15min":0.33700000000000002},"min":{"1min":0.251,"5min":0.216,"15min":0.216},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.372,"5min":0.38200000000000001,"15min":0.34999999999999998},"min":{"1min":0.23599999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.51100000000000001,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.28599999999999998}]},{"osd":2,"last update":"Fri Mar 29 04:52:05 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.39500000000000002,"15min":0.38300000000000001},"min":{"1min":0.27200000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.48699999999999999,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.44600000000000001,"15min":0.432},"min":{"1min":0.27700000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.58899999999999997,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.35799999999999998}]},{"osd":3,"last update":"Fri Mar 29 04:52:37 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.437,"15min":0.45000000000000001},"min":{"1min":0.28499999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.42499999999999999,"15min":0.441},"min":{"1min":0.27600000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.36799999999999999}]},{"osd":4,"last update":"Fri Mar 29 04:51:52 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.56599999999999995,"15min":0.57199999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.59099999999999997,"15min":0.59199999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.53800000000000003}]},{"osd":5,"last update":"Fri Mar 29 04:52:19 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57299999999999995,"15min":0.57099999999999995},"min":{"1min":0.40300000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.57399999999999995,"15min":0.57999999999999996},"min":{"1min":0.40500000000000003,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.73699999999999999,"5min":0.749,"15min":0.749},"last":0.51900000000000002}]},{"osd":6,"last update":"Fri Mar 29 04:52:39 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.57499999999999996,"15min":0.57899999999999996},"min":{"1min":0.42599999999999999,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.72299999999999998,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.441},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.59399999999999997,"15min":0.60399999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.498}]},{"osd":7,"last update":"Fri Mar 29 04:52:05 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.58399999999999996,"15min":0.58499999999999996},"min":{"1min":0.432,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.755,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.58199999999999996,"15min":0.58599999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.77900000000000003,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.58799999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298652,"kb_used_data":1292,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446500,"statfs":{"total":95995035648,"available":95689216000,"internally_reserved":0,"allocated":1323008,"data_stored":907164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Mar 29 04:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.38200000000000001,"15min":0.36799999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.54900000000000004,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.38400000000000001,"15min":0.38200000000000001},"min":{"1min":0.25800000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.54800000000000004,"5min":0.621,"15min":0.621},"last":0.42599999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:52:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40899999999999997,"15min":0.40300000000000002},"min":{"1min":0.28899999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.54400000000000004,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.40799999999999997,"15min":0.40699999999999997},"min":{"1min":0.28899999999999998,"5min":0.252,"15min":0.252},"max":{"1min":0.56000000000000005,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.40999999999999998}]},{"osd":3,"last update":"Fri Mar 29 04:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.439,"15min":0.45100000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.57599999999999996,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.41799999999999998,"15min":0.432},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.50700000000000001,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.47799999999999998}]},{"osd":4,"last update":"Fri Mar 29 04:51:54 2024","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.59499999999999997,"15min":0.59299999999999997},"min":{"1min":0.46500000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.57999999999999996,"15min":0.57499999999999996},"min":{"1min":0.40500000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.54900000000000004}]},{"osd":5,"last update":"Fri Mar 29 04:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.58499999999999996,"15min":0.58199999999999996},"min":{"1min":0.39200000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.56000000000000005,"15min":0.55400000000000005},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.61699999999999999}]},{"osd":6,"last update":"Fri Mar 29 04:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.55900000000000005,"15min":0.54800000000000004},"min":{"1min":0.38800000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.61299999999999999,"15min":0.61799999999999999},"min":{"1min":0.46200000000000002,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.60499999999999998}]},{"osd":7,"last update":"Fri Mar 29 04:52:04 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.434,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.80400000000000005,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.501},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.58799999999999997,"15min":0.58399999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77500000000000002,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.58399999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411390,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":729088,"data_stored":316808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:52:05 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.36299999999999999,"15min":0.35199999999999998},"min":{"1min":0.23400000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.48099999999999998,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.37},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.38800000000000001,"15min":0.36899999999999999},"min":{"1min":0.25800000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.35199999999999998}]},{"osd":1,"last update":"Fri Mar 29 04:52:05 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.38600000000000001,"15min":0.379},"min":{"1min":0.29099999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.502,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.40200000000000002,"15min":0.38900000000000001},"min":{"1min":0.26100000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.497,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.439}]},{"osd":3,"last update":"Fri Mar 29 04:52:32 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.53700000000000003,"5min":1.232,"15min":1.232},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.44700000000000001,"15min":0.47799999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.49399999999999999,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.33900000000000002}]},{"osd":4,"last update":"Fri Mar 29 04:51:50 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.59999999999999998,"15min":0.59599999999999997},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.82699999999999996,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.627,"15min":0.624},"min":{"1min":0.44800000000000001,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.79700000000000004,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.49199999999999999}]},{"osd":5,"last update":"Fri Mar 29 04:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59299999999999997,"15min":0.60299999999999998},"min":{"1min":0.42499999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.622,"15min":0.63700000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.84099999999999997,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.56899999999999995}]},{"osd":6,"last update":"Fri Mar 29 04:52:49 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.60899999999999999,"15min":0.625},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.82299999999999995,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.61899999999999999,"15min":0.63500000000000001},"min":{"1min":0.38500000000000001,"5min":0.375,"15min":0.375},"max":{"1min":0.73599999999999999,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.59699999999999998}]},{"osd":7,"last update":"Fri Mar 29 04:52:05 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.57299999999999995,"15min":0.57099999999999995},"min":{"1min":0.435,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.63600000000000001,"15min":0.64000000000000001},"min":{"1min":0.45100000000000001,"5min":0.44,"15min":0.44},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.51400000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149753,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":729088,"data_stored":316559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:52:32 2024","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.378,"15min":0.40400000000000003},"min":{"1min":0.24299999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.47999999999999998,"5min":0.53600000000000003,"15min":0.53600000000000003},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.40799999999999997,"15min":0.42899999999999999},"min":{"1min":0.253,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.56799999999999995,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.29699999999999999}]},{"osd":1,"last update":"Fri Mar 29 04:52:32 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.38600000000000001,"15min":0.40200000000000002},"min":{"1min":0.29899999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.48099999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.371,"5min":0.42999999999999999,"15min":0.45200000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.53900000000000003,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.442}]},{"osd":2,"last update":"Fri Mar 29 04:52:32 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.439,"15min":0.45700000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.48599999999999999,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.374,"5min":0.433,"15min":0.47899999999999998},"min":{"1min":0.26000000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.505,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.34200000000000003}]},{"osd":4,"last update":"Fri Mar 29 04:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.60399999999999998,"15min":0.61799999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.746,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58499999999999996,"15min":0.58899999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.71399999999999997}]},{"osd":5,"last update":"Fri Mar 29 04:52:19 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60699999999999998,"15min":0.61699999999999999},"min":{"1min":0.42699999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.75600000000000001,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.60599999999999998,"15min":0.61599999999999999},"min":{"1min":0.44500000000000001,"5min":0.434,"15min":0.434},"max":{"1min":0.69699999999999995,"5min":1.042,"15min":1.042},"last":0.69299999999999995}]},{"osd":6,"last update":"Fri Mar 29 04:52:49 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.437,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79100000000000004,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60699999999999998,"15min":0.61299999999999999},"min":{"1min":0.46600000000000003,"5min":0.373,"15min":0.373},"max":{"1min":0.73599999999999999,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.53700000000000003}]},{"osd":7,"last update":"Fri Mar 29 04:52:08 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.46200000000000002,"5min":0.436,"15min":0.436},"max":{"1min":0.77200000000000002,"5min":1.002,"15min":1.002},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.621,"5min":0.60099999999999998,"15min":0.59799999999999998},"min":{"1min":0.496,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.83199999999999996,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.57299999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986227,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298084,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447068,"statfs":{"total":95995035648,"available":95689797632,"internally_reserved":0,"allocated":741376,"data_stored":317637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:51:54 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.54000000000000004,"15min":0.54400000000000004},"min":{"1min":0.379,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.55400000000000005,"15min":0.56599999999999995},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.70199999999999996,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.69799999999999995}]},{"osd":1,"last update":"Fri Mar 29 04:51:54 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.53300000000000003,"15min":0.53600000000000003},"min":{"1min":0.32500000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.73799999999999999,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.54400000000000004,"15min":0.55000000000000004},"min":{"1min":0.39800000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.77200000000000002,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.56799999999999995}]},{"osd":2,"last update":"Fri Mar 29 04:51:54 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.54800000000000004,"15min":0.55800000000000005},"min":{"1min":0.36499999999999999,"5min":0.312,"15min":0.312},"max":{"1min":0.749,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.498},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.53800000000000003,"15min":0.54100000000000004},"min":{"1min":0.38800000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.80400000000000005,"5min":0.877,"15min":0.877},"last":0.73799999999999999}]},{"osd":3,"last update":"Fri Mar 29 04:51:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.55000000000000004,"15min":0.55300000000000005},"min":{"1min":0.379,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.73699999999999999,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.56499999999999995,"15min":0.57099999999999995},"min":{"1min":0.436,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.78300000000000003,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.52700000000000002}]},{"osd":5,"last update":"Fri Mar 29 04:52:13 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.436,"15min":0.42199999999999999},"min":{"1min":0.249,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.69699999999999995,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.49299999999999999,"15min":0.47799999999999998},"min":{"1min":0.33000000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.71699999999999997,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.65200000000000002}]},{"osd":6,"last update":"Fri Mar 29 04:52:39 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.47899999999999998,"15min":0.47599999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.65200000000000002,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.503,"5min":0.48199999999999998,"15min":0.47499999999999998},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.64900000000000002,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.623}]},{"osd":7,"last update":"Fri Mar 29 04:52:09 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.63100000000000001,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.53500000000000003,"15min":0.53300000000000003},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.72899999999999998,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.63800000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724590,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":729088,"data_stored":316786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.53100000000000003,"15min":0.54100000000000004},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.68100000000000005,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.53200000000000003,"15min":0.55000000000000004},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.66900000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.57599999999999996}]},{"osd":1,"last update":"Fri Mar 29 04:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.53500000000000003,"15min":0.54500000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.66500000000000004,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.54700000000000004,"15min":0.55300000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.64500000000000002,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.48399999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.57399999999999995,"15min":0.58199999999999996},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.74399999999999999,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.53900000000000003,"15min":0.54400000000000004},"min":{"1min":0.36399999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.66800000000000004,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.41099999999999998}]},{"osd":3,"last update":"Fri Mar 29 04:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.53400000000000003,"15min":0.54000000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.753,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.497},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.57199999999999995,"15min":0.58199999999999996},"min":{"1min":0.40799999999999997,"5min":0.37,"15min":0.37},"max":{"1min":0.71099999999999997,"5min":0.753,"15min":0.753},"last":0.433}]},{"osd":4,"last update":"Fri Mar 29 04:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.42499999999999999,"15min":0.39600000000000002},"min":{"1min":0.32300000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.70099999999999996,"5min":1.075,"15min":1.075},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.442,"5min":0.438,"15min":0.42099999999999999},"min":{"1min":0.30099999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.59399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.44700000000000001}]},{"osd":6,"last update":"Fri Mar 29 04:52:43 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.47699999999999998,"15min":0.46100000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.63200000000000001,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.48399999999999999,"15min":0.45200000000000001},"min":{"1min":0.373,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.47099999999999997}]},{"osd":7,"last update":"Fri Mar 29 04:52:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.55100000000000005,"15min":0.54700000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.73099999999999998,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.55400000000000005,"15min":0.55200000000000005},"min":{"1min":0.40400000000000003,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78900000000000003,"5min":1.115,"15min":1.115},"last":0.54800000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T04:52:54.061 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-29T04:52:54.062 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-29T04:52:54.062 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-29T04:52:54.062 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:52:54.316 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:54.700 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:52:54 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:52:54] "GET /metrics HTTP/1.1" 200 239123 "" "Prometheus/2.43.0" 2024-03-29T04:52:55.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:54 smithi181 ceph-mon[29765]: from='client.24680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:52:55.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:54 smithi181 ceph-mon[29765]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:55.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:54 smithi181 ceph-mon[41183]: from='client.24680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:52:55.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:54 smithi181 ceph-mon[41183]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:55.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:54 smithi187 ceph-mon[33595]: from='client.24680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:52:55.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:54 smithi187 ceph-mon[33595]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:56.383 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T04:52:56.383 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-29T04:52:57.082 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:56 smithi181 ceph-mon[29765]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:57.082 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:56 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3180710607' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:57.082 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:56 smithi181 ceph-mon[41183]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:57.082 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:56 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3180710607' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:57.110 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-03-29T04:52:57.110 INFO:teuthology.run_tasks:Running task print... 2024-03-29T04:52:57.124 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-03-29T04:52:57.124 INFO:teuthology.run_tasks:Running task parallel... 2024-03-29T04:52:57.135 INFO:teuthology.task.parallel:starting parallel... 2024-03-29T04:52:57.135 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-29T04:52:57.136 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-03-29T04:52:57.140 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-03-29T04:52:57.140 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-03-29T04:52:57.141 DEBUG:teuthology.orchestra.run.smithi181:> stat -- /home/ubuntu/cephtest/mnt.0 2024-03-29T04:52:57.141 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-29T04:52:57.141 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-03-29T04:52:57.142 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi181.front.sepia.ceph.com 2024-03-29T04:52:57.142 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-03-29T04:52:57.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:52:57.158 INFO:teuthology.orchestra.run.smithi181.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-03-29T04:52:57.159 DEBUG:teuthology.orchestra.run.smithi181:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-03-29T04:52:57.213 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-03-29T04:52:57.214 DEBUG:teuthology.orchestra.run.smithi181:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-03-29T04:52:57.268 INFO:tasks.workunit:timeout=3h 2024-03-29T04:52:57.268 INFO:tasks.workunit:cleanup=True 2024-03-29T04:52:57.269 DEBUG:teuthology.orchestra.run.smithi181:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-03-29T04:52:57.325 INFO:tasks.workunit.client.0.smithi181.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-03-29T04:52:57.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:56 smithi187 ceph-mon[33595]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:57.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:56 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3180710607' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:57.409 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:58.881 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:52:58 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:52:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:52:59.149 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:52:58 smithi187 ceph-mon[33595]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:59.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:52:58 smithi181 ceph-mon[29765]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:59.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:52:58 smithi181 ceph-mon[41183]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:52:59.679 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:52:59 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:52:59] ENGINE Bus STOPPING 2024-03-29T04:52:59.680 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:52:59 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:52:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:52:59.680 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:52:59 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:52:59] ENGINE Bus STOPPED 2024-03-29T04:52:59.680 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:52:59 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:52:59] ENGINE Bus STARTING 2024-03-29T04:52:59.680 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:52:59 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:52:59] ENGINE Serving on http://:::9283 2024-03-29T04:52:59.680 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:52:59 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:52:59] ENGINE Bus STARTED 2024-03-29T04:53:00.146 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph config set global log_to_journald false --force' 2024-03-29T04:53:00.442 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:00.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:00 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' 2024-03-29T04:53:00.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:00 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:53:00.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:00 smithi181 ceph-mon[29765]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:00.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:00 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:53:00.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:00 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:53:00.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:00 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:53:00.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:00 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' 2024-03-29T04:53:00.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:00 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:53:00.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:00 smithi181 ceph-mon[41183]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:00.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:00 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:53:00.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:00 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:53:00.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:00 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:53:00.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:00 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' 2024-03-29T04:53:00.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:00 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:53:00.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:00 smithi187 ceph-mon[33595]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:00.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:00 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:53:00.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:00 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:53:00.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:00 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:53:02.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:02 smithi181 ceph-mon[29765]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:02.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:02 smithi181 ceph-mon[41183]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:03.127 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-03-29T04:53:03.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:02 smithi187 ceph-mon[33595]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:03.383 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:03.790 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:03 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:03.790 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:03 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:04.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:03 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:04.916 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:04 smithi181 ceph-mon[29765]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:04.916 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:53:04 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:53:04] "GET /metrics HTTP/1.1" 200 239108 "" "Prometheus/2.43.0" 2024-03-29T04:53:04.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:04 smithi181 ceph-mon[41183]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:05.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:04 smithi187 ceph-mon[33595]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:05.356 INFO:teuthology.orchestra.run.smithi181.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:53:06.032 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T04:53:06.301 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:06.613 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[41183]: from='client.24704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:53:06.613 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[41183]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:53:06.613 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:53:06.613 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:53:06.613 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:53:06.613 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:53:06.614 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:53:06.614 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[41183]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:53:06.614 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[41183]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:06.614 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[29765]: from='client.24704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:53:06.614 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[29765]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:53:06.614 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:53:06.614 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:53:06.614 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:53:06.614 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:53:06.614 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:53:06.614 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[29765]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:53:06.614 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:06 smithi181 ceph-mon[29765]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:06.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:06 smithi187 ceph-mon[33595]: from='client.24704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:53:06.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:06 smithi187 ceph-mon[33595]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:53:06.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:06 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:53:06.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:06 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:53:06.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:06 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:53:06.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:06 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:53:06.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:06 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:53:06.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:06 smithi187 ceph-mon[33595]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:53:06.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:06 smithi187 ceph-mon[33595]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:08.419 INFO:teuthology.orchestra.run.smithi181.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T04:53:08.419 INFO:teuthology.orchestra.run.smithi181.stdout:alertmanager.a smithi181 *:9093,9094 running (81s) 19s ago 2m 15.4M - 0.25.0 c8568f914cd2 d22dcca4e726 2024-03-29T04:53:08.419 INFO:teuthology.orchestra.run.smithi181.stdout:grafana.a smithi187 *:3000 running (2m) 52s ago 2m 84.2M - 9.4.7 954c08fa6188 8bb6c690e335 2024-03-29T04:53:08.420 INFO:teuthology.orchestra.run.smithi181.stdout:mds.foo.smithi181.pflrwm smithi181 running (65s) 19s ago 65s 12.3M - 17.2.7-789-g4213e17c cf22eb8683be 0f200e33dce6 2024-03-29T04:53:08.420 INFO:teuthology.orchestra.run.smithi181.stdout:mds.foo.smithi187.vcsnqx smithi187 running (63s) 52s ago 62s 14.4M - 17.2.7-789-g4213e17c cf22eb8683be 245621ed14ed 2024-03-29T04:53:08.420 INFO:teuthology.orchestra.run.smithi181.stdout:mgr.x smithi187 *:8443 running (6m) 52s ago 6m 431M - 17.2.7-789-g4213e17c cf22eb8683be f668da78a625 2024-03-29T04:53:08.420 INFO:teuthology.orchestra.run.smithi181.stdout:mgr.y smithi181 *:9283 running (8m) 19s ago 8m 476M - 17.2.7-789-g4213e17c cf22eb8683be 37fa586e0972 2024-03-29T04:53:08.420 INFO:teuthology.orchestra.run.smithi181.stdout:mon.a smithi181 running (8m) 19s ago 8m 45.9M 2048M 17.2.7-789-g4213e17c cf22eb8683be 3e3936015cad 2024-03-29T04:53:08.420 INFO:teuthology.orchestra.run.smithi181.stdout:mon.b smithi187 running (6m) 52s ago 6m 39.7M 2048M 17.2.7-789-g4213e17c cf22eb8683be a376bfeefb23 2024-03-29T04:53:08.420 INFO:teuthology.orchestra.run.smithi181.stdout:mon.c smithi181 running (6m) 19s ago 6m 35.9M 2048M 17.2.7-789-g4213e17c cf22eb8683be 16379529f880 2024-03-29T04:53:08.420 INFO:teuthology.orchestra.run.smithi181.stdout:node-exporter.a smithi181 *:9100 running (114s) 19s ago 112s 15.3M - 1.5.0 0da6a335fe13 2f1dfb6435e1 2024-03-29T04:53:08.420 INFO:teuthology.orchestra.run.smithi181.stdout:node-exporter.b smithi187 *:9100 running (105s) 52s ago 105s 7956k - 1.5.0 0da6a335fe13 9528215f6d2a 2024-03-29T04:53:08.420 INFO:teuthology.orchestra.run.smithi181.stdout:osd.0 smithi181 running (6m) 19s ago 6m 42.5M 4016M 17.2.7-789-g4213e17c cf22eb8683be ae426d29a1a8 2024-03-29T04:53:08.420 INFO:teuthology.orchestra.run.smithi181.stdout:osd.1 smithi181 running (5m) 19s ago 5m 41.7M 4016M 17.2.7-789-g4213e17c cf22eb8683be 7ed1966c65cf 2024-03-29T04:53:08.420 INFO:teuthology.orchestra.run.smithi181.stdout:osd.2 smithi181 running (5m) 19s ago 5m 41.2M 4016M 17.2.7-789-g4213e17c cf22eb8683be 9c61fb0d684c 2024-03-29T04:53:08.420 INFO:teuthology.orchestra.run.smithi181.stdout:osd.3 smithi181 running (4m) 19s ago 4m 43.3M 4016M 17.2.7-789-g4213e17c cf22eb8683be 6b0bf471cfcb 2024-03-29T04:53:08.420 INFO:teuthology.orchestra.run.smithi181.stdout:osd.4 smithi187 running (4m) 52s ago 4m 41.0M 4016M 17.2.7-789-g4213e17c cf22eb8683be 42698769ef84 2024-03-29T04:53:08.420 INFO:teuthology.orchestra.run.smithi181.stdout:osd.5 smithi187 running (4m) 52s ago 4m 42.5M 4016M 17.2.7-789-g4213e17c cf22eb8683be 9599ff85a9e7 2024-03-29T04:53:08.421 INFO:teuthology.orchestra.run.smithi181.stdout:osd.6 smithi187 running (3m) 52s ago 3m 42.0M 4016M 17.2.7-789-g4213e17c cf22eb8683be c3287470c1fa 2024-03-29T04:53:08.421 INFO:teuthology.orchestra.run.smithi181.stdout:osd.7 smithi187 running (3m) 52s ago 3m 40.5M 4016M 17.2.7-789-g4213e17c cf22eb8683be 89fb8ccf8705 2024-03-29T04:53:08.421 INFO:teuthology.orchestra.run.smithi181.stdout:prometheus.a smithi187 *:9095 running (75s) 52s ago 2m 35.6M - 2.43.0 a07b618ecd1d f7620d3fb68f 2024-03-29T04:53:08.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:08 smithi187 ceph-mon[33595]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:08.881 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:53:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:53:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:53:08.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:08 smithi181 ceph-mon[29765]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:08.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:08 smithi181 ceph-mon[41183]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:09.417 DEBUG:teuthology.orchestra.run.smithi181:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-03-29T04:53:09.549 INFO:tasks.workunit.client.0.smithi181.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-03-29T04:53:09.551 INFO:tasks.workunit.client.0.smithi181.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-03-29T04:53:09.552 INFO:tasks.workunit.client.0.smithi181.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-03-29T04:53:09.574 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'echo wait for minority of mons to upgrade' 2024-03-29T04:53:09.681 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:09 smithi181 ceph-mon[29765]: from='client.24710 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:53:09.864 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:09.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:09 smithi187 ceph-mon[33595]: from='client.24710 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:53:09.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:09 smithi181 ceph-mon[41183]: from='client.24710 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:53:10.683 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:10 smithi181 ceph-mon[41183]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:10.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:10 smithi181 ceph-mon[29765]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:11.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:10 smithi187 ceph-mon[33595]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:11.436 INFO:tasks.workunit.client.0.smithi181.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-03-29T04:53:11.530 INFO:tasks.workunit.client.0.smithi181.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-03-29T04:53:11.586 INFO:tasks.workunit.client.0.smithi181.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-03-29T04:53:11.588 INFO:tasks.workunit.client.0.smithi181.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-03-29T04:53:11.588 INFO:tasks.workunit.client.0.smithi181.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-03-29T04:53:11.635 INFO:tasks.workunit.client.0.smithi181.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-03-29T04:53:11.638 DEBUG:teuthology.orchestra.run.smithi181:> set -ex 2024-03-29T04:53:11.638 DEBUG:teuthology.orchestra.run.smithi181:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-03-29T04:53:11.658 INFO:tasks.workunit:Running workunits matching rbd/test_librbd.sh on client.0... 2024-03-29T04:53:11.659 INFO:tasks.workunit:Running workunit rbd/test_librbd.sh... 2024-03-29T04:53:11.659 DEBUG:teuthology.orchestra.run.smithi181:workunit test rbd/test_librbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd.sh 2024-03-29T04:53:11.767 INFO:tasks.workunit.client.0.smithi181.stderr:warning: failed to enable lockdep 2024-03-29T04:53:11.768 INFO:tasks.workunit.client.0.smithi181.stdout:seed 86707 2024-03-29T04:53:11.768 INFO:tasks.workunit.client.0.smithi181.stdout:[==========] Running 415 tests from 16 test suites. 2024-03-29T04:53:11.768 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] Global test environment set-up. 2024-03-29T04:53:11.768 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 1 test from TestJournalStress 2024-03-29T04:53:12.183 INFO:teuthology.orchestra.run.smithi181.stdout:wait for minority of mons to upgrade 2024-03-29T04:53:12.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:12 smithi181 ceph-mon[41183]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:12.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:12 smithi181 ceph-mon[29765]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:12.979 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-03-29T04:53:13.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:12 smithi187 ceph-mon[33595]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:13.265 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:13.709 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalStress.DiscardWithPruneWriteOverlap 2024-03-29T04:53:13.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:13 smithi181 ceph-mon[29765]: osdmap e72: 8 total, 8 up, 8 in 2024-03-29T04:53:13.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:13 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/934726192' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:13.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:13 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:13.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:13 smithi181 ceph-mon[41183]: osdmap e72: 8 total, 8 up, 8 in 2024-03-29T04:53:13.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:13 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/934726192' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:13.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:13 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:14.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:13 smithi187 ceph-mon[33595]: osdmap e72: 8 total, 8 up, 8 in 2024-03-29T04:53:14.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:13 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/934726192' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:14.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:13 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:14.909 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:53:14 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:53:14] "GET /metrics HTTP/1.1" 200 239108 "" "Prometheus/2.43.0" 2024-03-29T04:53:14.909 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:14 smithi181 ceph-mon[41183]: pgmap v95: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:14.909 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:14 smithi181 ceph-mon[41183]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:53:14.909 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:14 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:14.909 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:14 smithi181 ceph-mon[41183]: osdmap e73: 8 total, 8 up, 8 in 2024-03-29T04:53:14.910 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:14 smithi181 ceph-mon[29765]: pgmap v95: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:14.910 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:14 smithi181 ceph-mon[29765]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:53:14.910 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:14 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:14.910 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:14 smithi181 ceph-mon[29765]: osdmap e73: 8 total, 8 up, 8 in 2024-03-29T04:53:15.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:14 smithi187 ceph-mon[33595]: pgmap v95: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:15.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:14 smithi187 ceph-mon[33595]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:53:15.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:14 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:15.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:14 smithi187 ceph-mon[33595]: osdmap e73: 8 total, 8 up, 8 in 2024-03-29T04:53:15.732 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:15.732 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalStress.DiscardWithPruneWriteOverlap (2023 ms) 2024-03-29T04:53:16.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:15 smithi187 ceph-mon[33595]: osdmap e74: 8 total, 8 up, 8 in 2024-03-29T04:53:16.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:15 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1037413461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:16.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:15 smithi181 ceph-mon[29765]: osdmap e74: 8 total, 8 up, 8 in 2024-03-29T04:53:16.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:15 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1037413461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:16.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:15 smithi181 ceph-mon[41183]: osdmap e74: 8 total, 8 up, 8 in 2024-03-29T04:53:16.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:15 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1037413461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:16.719 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 1 test from TestJournalStress (2023 ms total) 2024-03-29T04:53:16.719 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T04:53:16.719 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 16 tests from TestJournalReplay 2024-03-29T04:53:17.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:16 smithi187 ceph-mon[33595]: pgmap v98: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:17.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:16 smithi187 ceph-mon[33595]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:53:17.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:16 smithi187 ceph-mon[33595]: Cluster is now healthy 2024-03-29T04:53:17.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:16 smithi187 ceph-mon[33595]: osdmap e75: 8 total, 8 up, 8 in 2024-03-29T04:53:17.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:16 smithi181 ceph-mon[41183]: pgmap v98: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:17.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:16 smithi181 ceph-mon[41183]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:53:17.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:16 smithi181 ceph-mon[41183]: Cluster is now healthy 2024-03-29T04:53:17.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:16 smithi181 ceph-mon[41183]: osdmap e75: 8 total, 8 up, 8 in 2024-03-29T04:53:17.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:16 smithi181 ceph-mon[29765]: pgmap v98: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:17.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:16 smithi181 ceph-mon[29765]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:53:17.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:16 smithi181 ceph-mon[29765]: Cluster is now healthy 2024-03-29T04:53:17.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:16 smithi181 ceph-mon[29765]: osdmap e75: 8 total, 8 up, 8 in 2024-03-29T04:53:18.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:17 smithi187 ceph-mon[33595]: osdmap e76: 8 total, 8 up, 8 in 2024-03-29T04:53:18.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:17 smithi181 ceph-mon[29765]: osdmap e76: 8 total, 8 up, 8 in 2024-03-29T04:53:18.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:17 smithi181 ceph-mon[41183]: osdmap e76: 8 total, 8 up, 8 in 2024-03-29T04:53:18.729 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalReplay.AioDiscardEvent 2024-03-29T04:53:18.881 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:53:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:53:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:53:18.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:18 smithi187 ceph-mon[33595]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:18.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:18 smithi187 ceph-mon[33595]: osdmap e77: 8 total, 8 up, 8 in 2024-03-29T04:53:18.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:18 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3733889106' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:18.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:18 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/8847780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:18.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:18 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:19.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:18 smithi181 ceph-mon[29765]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:19.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:18 smithi181 ceph-mon[29765]: osdmap e77: 8 total, 8 up, 8 in 2024-03-29T04:53:19.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:18 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3733889106' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:19.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:18 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/8847780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:19.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:18 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:19.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:18 smithi181 ceph-mon[41183]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:19.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:18 smithi181 ceph-mon[41183]: osdmap e77: 8 total, 8 up, 8 in 2024-03-29T04:53:19.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:18 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3733889106' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:19.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:18 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/8847780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:19.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:18 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:20.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:19 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3733889106' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:20.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:19 smithi187 ceph-mon[33595]: osdmap e78: 8 total, 8 up, 8 in 2024-03-29T04:53:20.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:19 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3733889106' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:20.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:19 smithi181 ceph-mon[41183]: osdmap e78: 8 total, 8 up, 8 in 2024-03-29T04:53:20.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:19 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3733889106' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:20.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:19 smithi181 ceph-mon[29765]: osdmap e78: 8 total, 8 up, 8 in 2024-03-29T04:53:20.761 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:20.761 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalReplay.AioDiscardEvent (2032 ms) 2024-03-29T04:53:20.761 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalReplay.AioWriteEvent 2024-03-29T04:53:20.770 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:20.770 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalReplay.AioWriteEvent (9 ms) 2024-03-29T04:53:20.770 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalReplay.AioFlushEvent 2024-03-29T04:53:20.778 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:20.778 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalReplay.AioFlushEvent (8 ms) 2024-03-29T04:53:20.778 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalReplay.SnapCreate 2024-03-29T04:53:20.786 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:20.786 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalReplay.SnapCreate (8 ms) 2024-03-29T04:53:20.786 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalReplay.SnapProtect 2024-03-29T04:53:20.795 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:20.795 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalReplay.SnapProtect (9 ms) 2024-03-29T04:53:20.795 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalReplay.SnapUnprotect 2024-03-29T04:53:20.804 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:20.804 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalReplay.SnapUnprotect (9 ms) 2024-03-29T04:53:20.804 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalReplay.SnapRename 2024-03-29T04:53:20.814 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:20.814 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalReplay.SnapRename (10 ms) 2024-03-29T04:53:20.814 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalReplay.SnapRollback 2024-03-29T04:53:20.823 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:20.823 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalReplay.SnapRollback (9 ms) 2024-03-29T04:53:20.823 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalReplay.SnapRemove 2024-03-29T04:53:20.833 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:20.833 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalReplay.SnapRemove (10 ms) 2024-03-29T04:53:20.833 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalReplay.Rename 2024-03-29T04:53:20.841 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:20.841 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalReplay.Rename (8 ms) 2024-03-29T04:53:20.841 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalReplay.Resize 2024-03-29T04:53:20.849 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:20.849 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalReplay.Resize (8 ms) 2024-03-29T04:53:20.849 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalReplay.Flatten 2024-03-29T04:53:20.857 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:20.857 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalReplay.Flatten (8 ms) 2024-03-29T04:53:20.857 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalReplay.UpdateFeatures 2024-03-29T04:53:20.864 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:20.864 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalReplay.UpdateFeatures (7 ms) 2024-03-29T04:53:20.864 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalReplay.MetadataSet 2024-03-29T04:53:20.871 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:20.872 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalReplay.MetadataSet (8 ms) 2024-03-29T04:53:20.872 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalReplay.MetadataRemove 2024-03-29T04:53:20.881 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:20.881 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalReplay.MetadataRemove (7 ms) 2024-03-29T04:53:20.881 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalReplay.ObjectPosition 2024-03-29T04:53:20.886 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:20.886 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalReplay.ObjectPosition (7 ms) 2024-03-29T04:53:21.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:20 smithi187 ceph-mon[33595]: pgmap v104: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:21.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:20 smithi187 ceph-mon[33595]: osdmap e79: 8 total, 8 up, 8 in 2024-03-29T04:53:21.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:20 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/14516882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:21.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:20 smithi181 ceph-mon[41183]: pgmap v104: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:21.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:20 smithi181 ceph-mon[41183]: osdmap e79: 8 total, 8 up, 8 in 2024-03-29T04:53:21.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:20 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/14516882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:21.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:20 smithi181 ceph-mon[29765]: pgmap v104: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:21.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:20 smithi181 ceph-mon[29765]: osdmap e79: 8 total, 8 up, 8 in 2024-03-29T04:53:21.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:20 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/14516882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:21.754 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 16 tests from TestJournalReplay (2157 ms total) 2024-03-29T04:53:21.754 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T04:53:21.754 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 4 tests from TestJournalEntries 2024-03-29T04:53:22.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:21 smithi187 ceph-mon[33595]: osdmap e80: 8 total, 8 up, 8 in 2024-03-29T04:53:22.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:21 smithi181 ceph-mon[41183]: osdmap e80: 8 total, 8 up, 8 in 2024-03-29T04:53:22.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:21 smithi181 ceph-mon[29765]: osdmap e80: 8 total, 8 up, 8 in 2024-03-29T04:53:23.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:22 smithi187 ceph-mon[33595]: pgmap v107: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:23.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:22 smithi187 ceph-mon[33595]: osdmap e81: 8 total, 8 up, 8 in 2024-03-29T04:53:23.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:22 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2117530726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:23.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:22 smithi181 ceph-mon[29765]: pgmap v107: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:23.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:22 smithi181 ceph-mon[29765]: osdmap e81: 8 total, 8 up, 8 in 2024-03-29T04:53:23.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:22 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2117530726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:23.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:22 smithi181 ceph-mon[41183]: pgmap v107: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:23.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:22 smithi181 ceph-mon[41183]: osdmap e81: 8 total, 8 up, 8 in 2024-03-29T04:53:23.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:22 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2117530726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:23.764 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalEntries.AioWrite 2024-03-29T04:53:24.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:23 smithi187 ceph-mon[33595]: osdmap e82: 8 total, 8 up, 8 in 2024-03-29T04:53:24.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:23 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2995212293' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:24.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:23 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:24.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:23 smithi181 ceph-mon[29765]: osdmap e82: 8 total, 8 up, 8 in 2024-03-29T04:53:24.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:23 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2995212293' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:24.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:23 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:24.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:23 smithi181 ceph-mon[41183]: osdmap e82: 8 total, 8 up, 8 in 2024-03-29T04:53:24.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:23 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2995212293' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:24.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:23 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:24.950 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:53:24 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:53:24] "GET /metrics HTTP/1.1" 200 241423 "" "Prometheus/2.43.0" 2024-03-29T04:53:24.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:24 smithi181 ceph-mon[29765]: pgmap v110: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:24.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:24 smithi181 ceph-mon[29765]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:53:24.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:24 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:24.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:24 smithi181 ceph-mon[29765]: osdmap e83: 8 total, 8 up, 8 in 2024-03-29T04:53:24.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:24 smithi181 ceph-mon[41183]: pgmap v110: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:24.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:24 smithi181 ceph-mon[41183]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:53:24.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:24 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:24.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:24 smithi181 ceph-mon[41183]: osdmap e83: 8 total, 8 up, 8 in 2024-03-29T04:53:25.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:24 smithi187 ceph-mon[33595]: pgmap v110: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:25.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:24 smithi187 ceph-mon[33595]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:53:25.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:24 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:25.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:24 smithi187 ceph-mon[33595]: osdmap e83: 8 total, 8 up, 8 in 2024-03-29T04:53:25.787 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:25.787 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalEntries.AioWrite (2023 ms) 2024-03-29T04:53:25.787 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalEntries.AioDiscard 2024-03-29T04:53:25.798 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:25.798 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalEntries.AioDiscard (11 ms) 2024-03-29T04:53:25.798 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalEntries.AioDiscardWithPrune 2024-03-29T04:53:25.806 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:25.806 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalEntries.AioDiscardWithPrune (8 ms) 2024-03-29T04:53:25.806 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestJournalEntries.AioFlush 2024-03-29T04:53:25.815 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:25.815 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestJournalEntries.AioFlush (9 ms) 2024-03-29T04:53:26.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:25 smithi187 ceph-mon[33595]: osdmap e84: 8 total, 8 up, 8 in 2024-03-29T04:53:26.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:25 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/662500041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:26.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:25 smithi181 ceph-mon[29765]: osdmap e84: 8 total, 8 up, 8 in 2024-03-29T04:53:26.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:25 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/662500041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:26.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:25 smithi181 ceph-mon[41183]: osdmap e84: 8 total, 8 up, 8 in 2024-03-29T04:53:26.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:25 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/662500041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:26.788 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 4 tests from TestJournalEntries (2051 ms total) 2024-03-29T04:53:26.788 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T04:53:26.788 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 2 tests from TestTrash 2024-03-29T04:53:27.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:26 smithi187 ceph-mon[33595]: pgmap v113: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:27.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:26 smithi187 ceph-mon[33595]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:53:27.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:26 smithi187 ceph-mon[33595]: Cluster is now healthy 2024-03-29T04:53:27.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:26 smithi187 ceph-mon[33595]: osdmap e85: 8 total, 8 up, 8 in 2024-03-29T04:53:27.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:26 smithi181 ceph-mon[29765]: pgmap v113: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:27.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:26 smithi181 ceph-mon[29765]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:53:27.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:26 smithi181 ceph-mon[29765]: Cluster is now healthy 2024-03-29T04:53:27.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:26 smithi181 ceph-mon[29765]: osdmap e85: 8 total, 8 up, 8 in 2024-03-29T04:53:27.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:26 smithi181 ceph-mon[41183]: pgmap v113: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:27.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:26 smithi181 ceph-mon[41183]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:53:27.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:26 smithi181 ceph-mon[41183]: Cluster is now healthy 2024-03-29T04:53:27.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:26 smithi181 ceph-mon[41183]: osdmap e85: 8 total, 8 up, 8 in 2024-03-29T04:53:28.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:27 smithi187 ceph-mon[33595]: osdmap e86: 8 total, 8 up, 8 in 2024-03-29T04:53:28.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:27 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/30139493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:28.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:27 smithi181 ceph-mon[29765]: osdmap e86: 8 total, 8 up, 8 in 2024-03-29T04:53:28.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:27 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/30139493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:28.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:27 smithi181 ceph-mon[41183]: osdmap e86: 8 total, 8 up, 8 in 2024-03-29T04:53:28.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:27 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/30139493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:28.795 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestTrash.UserRemovingSource 2024-03-29T04:53:28.799 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:53:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:53:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:53:29.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:28 smithi187 ceph-mon[33595]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:29.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:28 smithi187 ceph-mon[33595]: osdmap e87: 8 total, 8 up, 8 in 2024-03-29T04:53:29.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:28 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2810022723' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:29.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:28 smithi181 ceph-mon[29765]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:29.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:28 smithi181 ceph-mon[29765]: osdmap e87: 8 total, 8 up, 8 in 2024-03-29T04:53:29.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:28 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2810022723' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:29.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:28 smithi181 ceph-mon[41183]: pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:29.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:28 smithi181 ceph-mon[41183]: osdmap e87: 8 total, 8 up, 8 in 2024-03-29T04:53:29.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:28 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2810022723' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:30.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:29 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2810022723' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:30.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:29 smithi187 ceph-mon[33595]: osdmap e88: 8 total, 8 up, 8 in 2024-03-29T04:53:30.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:29 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2810022723' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:30.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:29 smithi181 ceph-mon[29765]: osdmap e88: 8 total, 8 up, 8 in 2024-03-29T04:53:30.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:29 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2810022723' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:30.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:29 smithi181 ceph-mon[41183]: osdmap e88: 8 total, 8 up, 8 in 2024-03-29T04:53:30.898 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestTrash.UserRemovingSource (2103 ms) 2024-03-29T04:53:30.898 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestTrash.RestoreMirroringSource 2024-03-29T04:53:30.939 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestTrash.RestoreMirroringSource (41 ms) 2024-03-29T04:53:31.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:30 smithi187 ceph-mon[33595]: pgmap v119: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:31.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:30 smithi187 ceph-mon[33595]: osdmap e89: 8 total, 8 up, 8 in 2024-03-29T04:53:31.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:30 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1596426038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:31.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:30 smithi181 ceph-mon[41183]: pgmap v119: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:31.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:30 smithi181 ceph-mon[41183]: osdmap e89: 8 total, 8 up, 8 in 2024-03-29T04:53:31.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:30 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1596426038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:31.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:30 smithi181 ceph-mon[29765]: pgmap v119: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:31.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:30 smithi181 ceph-mon[29765]: osdmap e89: 8 total, 8 up, 8 in 2024-03-29T04:53:31.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:30 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1596426038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:31.831 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 2 tests from TestTrash (2144 ms total) 2024-03-29T04:53:31.831 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T04:53:31.831 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 1 test from TestOperations 2024-03-29T04:53:32.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:31 smithi187 ceph-mon[33595]: osdmap e90: 8 total, 8 up, 8 in 2024-03-29T04:53:32.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:31 smithi181 ceph-mon[29765]: osdmap e90: 8 total, 8 up, 8 in 2024-03-29T04:53:32.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:31 smithi181 ceph-mon[41183]: osdmap e90: 8 total, 8 up, 8 in 2024-03-29T04:53:33.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:32 smithi187 ceph-mon[33595]: pgmap v122: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:33.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:32 smithi187 ceph-mon[33595]: osdmap e91: 8 total, 8 up, 8 in 2024-03-29T04:53:33.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:32 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3946215318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:33.195 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:32 smithi181 ceph-mon[29765]: pgmap v122: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:33.195 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:32 smithi181 ceph-mon[29765]: osdmap e91: 8 total, 8 up, 8 in 2024-03-29T04:53:33.195 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:32 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3946215318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:33.195 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:32 smithi181 ceph-mon[41183]: pgmap v122: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:33.195 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:32 smithi181 ceph-mon[41183]: osdmap e91: 8 total, 8 up, 8 in 2024-03-29T04:53:33.196 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:32 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3946215318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:33.854 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestOperations.DisableJournalingCorrupt 2024-03-29T04:53:34.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:33 smithi187 ceph-mon[33595]: osdmap e92: 8 total, 8 up, 8 in 2024-03-29T04:53:34.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:33 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/4053713184' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:34.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:33 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:34.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:33 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:34.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:33 smithi181 ceph-mon[41183]: osdmap e92: 8 total, 8 up, 8 in 2024-03-29T04:53:34.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:33 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/4053713184' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:34.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:33 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:34.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:33 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:34.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:33 smithi181 ceph-mon[29765]: osdmap e92: 8 total, 8 up, 8 in 2024-03-29T04:53:34.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:33 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/4053713184' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:34.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:33 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:34.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:33 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:34.841 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:53:34 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:53:34] "GET /metrics HTTP/1.1" 200 241936 "" "Prometheus/2.43.0" 2024-03-29T04:53:34.841 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:34 smithi181 ceph-mon[29765]: pgmap v125: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:34.841 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:34 smithi181 ceph-mon[29765]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:53:35.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:34 smithi187 ceph-mon[33595]: pgmap v125: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:35.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:34 smithi187 ceph-mon[33595]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:53:35.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:34 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:35.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:34 smithi187 ceph-mon[33595]: osdmap e93: 8 total, 8 up, 8 in 2024-03-29T04:53:35.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:34 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1538121814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:35.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:34 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:35.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:34 smithi181 ceph-mon[29765]: osdmap e93: 8 total, 8 up, 8 in 2024-03-29T04:53:35.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:34 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1538121814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:35.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:34 smithi181 ceph-mon[41183]: pgmap v125: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:35.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:34 smithi181 ceph-mon[41183]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:53:35.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:34 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:35.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:34 smithi181 ceph-mon[41183]: osdmap e93: 8 total, 8 up, 8 in 2024-03-29T04:53:35.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:34 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1538121814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:35.873 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:35.873 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestOperations.DisableJournalingCorrupt (2018 ms) 2024-03-29T04:53:36.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:35 smithi187 ceph-mon[33595]: osdmap e94: 8 total, 8 up, 8 in 2024-03-29T04:53:36.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:35 smithi181 ceph-mon[41183]: osdmap e94: 8 total, 8 up, 8 in 2024-03-29T04:53:36.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:35 smithi181 ceph-mon[29765]: osdmap e94: 8 total, 8 up, 8 in 2024-03-29T04:53:36.863 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 1 test from TestOperations (2018 ms total) 2024-03-29T04:53:36.864 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T04:53:36.864 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 4 tests from TestObjectMap 2024-03-29T04:53:37.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:36 smithi187 ceph-mon[33595]: pgmap v128: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:37.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:36 smithi187 ceph-mon[33595]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:53:37.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:36 smithi187 ceph-mon[33595]: Cluster is now healthy 2024-03-29T04:53:37.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:36 smithi187 ceph-mon[33595]: osdmap e95: 8 total, 8 up, 8 in 2024-03-29T04:53:37.137 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:36 smithi181 ceph-mon[29765]: pgmap v128: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:37.137 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:36 smithi181 ceph-mon[29765]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:53:37.137 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:36 smithi181 ceph-mon[29765]: Cluster is now healthy 2024-03-29T04:53:37.137 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:36 smithi181 ceph-mon[29765]: osdmap e95: 8 total, 8 up, 8 in 2024-03-29T04:53:37.137 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:36 smithi181 ceph-mon[41183]: pgmap v128: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:37.138 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:36 smithi181 ceph-mon[41183]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:53:37.138 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:36 smithi181 ceph-mon[41183]: Cluster is now healthy 2024-03-29T04:53:37.138 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:36 smithi181 ceph-mon[41183]: osdmap e95: 8 total, 8 up, 8 in 2024-03-29T04:53:38.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:37 smithi187 ceph-mon[33595]: osdmap e96: 8 total, 8 up, 8 in 2024-03-29T04:53:38.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:37 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/379282451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:38.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:37 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:53:38.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:37 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:53:38.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:37 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:53:38.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:37 smithi181 ceph-mon[29765]: osdmap e96: 8 total, 8 up, 8 in 2024-03-29T04:53:38.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:37 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/379282451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:38.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:37 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:53:38.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:37 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:53:38.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:37 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:53:38.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:37 smithi181 ceph-mon[41183]: osdmap e96: 8 total, 8 up, 8 in 2024-03-29T04:53:38.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:37 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/379282451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:38.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:37 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:53:38.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:37 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:53:38.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:37 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:53:38.871 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenCorrupt 2024-03-29T04:53:38.874 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:53:38 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:53:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:53:39.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:38 smithi187 ceph-mon[33595]: Upgrade: Target is version 19.0.0-1959-g8da23624 (unknown) 2024-03-29T04:53:39.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:38 smithi187 ceph-mon[33595]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28'] 2024-03-29T04:53:39.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:38 smithi187 ceph-mon[33595]: Upgrade: Need to upgrade myself (mgr.y) 2024-03-29T04:53:39.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:38 smithi187 ceph-mon[33595]: pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:39.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:38 smithi187 ceph-mon[33595]: osdmap e97: 8 total, 8 up, 8 in 2024-03-29T04:53:39.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:38 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/438774511' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:39.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:38 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:39.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:38 smithi187 ceph-mon[33595]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 on smithi187 2024-03-29T04:53:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:38 smithi181 ceph-mon[29765]: Upgrade: Target is version 19.0.0-1959-g8da23624 (unknown) 2024-03-29T04:53:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:38 smithi181 ceph-mon[29765]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28'] 2024-03-29T04:53:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:38 smithi181 ceph-mon[29765]: Upgrade: Need to upgrade myself (mgr.y) 2024-03-29T04:53:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:38 smithi181 ceph-mon[29765]: pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:38 smithi181 ceph-mon[29765]: osdmap e97: 8 total, 8 up, 8 in 2024-03-29T04:53:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:38 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/438774511' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:38 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:38 smithi181 ceph-mon[29765]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 on smithi187 2024-03-29T04:53:39.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:38 smithi181 ceph-mon[41183]: Upgrade: Target is version 19.0.0-1959-g8da23624 (unknown) 2024-03-29T04:53:39.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:38 smithi181 ceph-mon[41183]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28'] 2024-03-29T04:53:39.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:38 smithi181 ceph-mon[41183]: Upgrade: Need to upgrade myself (mgr.y) 2024-03-29T04:53:39.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:38 smithi181 ceph-mon[41183]: pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:39.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:38 smithi181 ceph-mon[41183]: osdmap e97: 8 total, 8 up, 8 in 2024-03-29T04:53:39.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:38 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/438774511' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:39.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:38 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:39.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:38 smithi181 ceph-mon[41183]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 on smithi187 2024-03-29T04:53:40.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:39 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:40.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:39 smithi187 ceph-mon[33595]: osdmap e98: 8 total, 8 up, 8 in 2024-03-29T04:53:40.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:39 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/672422557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:40.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:39 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:40.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:39 smithi181 ceph-mon[41183]: osdmap e98: 8 total, 8 up, 8 in 2024-03-29T04:53:40.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:39 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/672422557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:40.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:39 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:40.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:39 smithi181 ceph-mon[29765]: osdmap e98: 8 total, 8 up, 8 in 2024-03-29T04:53:40.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:39 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/672422557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:40.913 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenCorrupt (2042 ms) 2024-03-29T04:53:40.913 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenTooSmall 2024-03-29T04:53:40.942 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenTooSmall (29 ms) 2024-03-29T04:53:40.942 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestObjectMap.InvalidateFlagOnDisk 2024-03-29T04:53:40.979 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestObjectMap.InvalidateFlagOnDisk (36 ms) 2024-03-29T04:53:40.979 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall 2024-03-29T04:53:41.002 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall (24 ms) 2024-03-29T04:53:41.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:40 smithi181 ceph-mon[29765]: pgmap v134: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:41.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:40 smithi181 ceph-mon[29765]: osdmap e99: 8 total, 8 up, 8 in 2024-03-29T04:53:41.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:40 smithi181 ceph-mon[41183]: pgmap v134: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:41.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:40 smithi181 ceph-mon[41183]: osdmap e99: 8 total, 8 up, 8 in 2024-03-29T04:53:41.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:40 smithi187 ceph-mon[33595]: pgmap v134: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:41.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:40 smithi187 ceph-mon[33595]: osdmap e99: 8 total, 8 up, 8 in 2024-03-29T04:53:41.893 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 4 tests from TestObjectMap (2132 ms total) 2024-03-29T04:53:41.893 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T04:53:41.893 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 2 tests from TestMirroringWatcher 2024-03-29T04:53:42.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:41 smithi181 ceph-mon[29765]: osdmap e100: 8 total, 8 up, 8 in 2024-03-29T04:53:42.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:41 smithi181 ceph-mon[41183]: osdmap e100: 8 total, 8 up, 8 in 2024-03-29T04:53:42.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:41 smithi187 ceph-mon[33595]: osdmap e100: 8 total, 8 up, 8 in 2024-03-29T04:53:43.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:42 smithi181 ceph-mon[29765]: pgmap v137: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:43.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:42 smithi181 ceph-mon[29765]: osdmap e101: 8 total, 8 up, 8 in 2024-03-29T04:53:43.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:42 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/860955126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:43.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:42 smithi181 ceph-mon[41183]: pgmap v137: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:43.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:42 smithi181 ceph-mon[41183]: osdmap e101: 8 total, 8 up, 8 in 2024-03-29T04:53:43.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:42 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/860955126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:43.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:42 smithi187 ceph-mon[33595]: pgmap v137: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:43.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:42 smithi187 ceph-mon[33595]: osdmap e101: 8 total, 8 up, 8 in 2024-03-29T04:53:43.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:42 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/860955126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:43.908 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroringWatcher.ModeUpdated 2024-03-29T04:53:44.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:43 smithi181 ceph-mon[29765]: osdmap e102: 8 total, 8 up, 8 in 2024-03-29T04:53:44.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:43 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2980023583' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:44.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:43 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:44.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:43 smithi181 ceph-mon[41183]: osdmap e102: 8 total, 8 up, 8 in 2024-03-29T04:53:44.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:43 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2980023583' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:44.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:43 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:44.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:43 smithi187 ceph-mon[33595]: osdmap e102: 8 total, 8 up, 8 in 2024-03-29T04:53:44.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:43 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2980023583' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:44.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:43 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:44.903 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:53:44 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:53:44] "GET /metrics HTTP/1.1" 200 241936 "" "Prometheus/2.43.0" 2024-03-29T04:53:45.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:44 smithi181 ceph-mon[41183]: pgmap v140: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:45.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:44 smithi181 ceph-mon[41183]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:53:45.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:44 smithi181 ceph-mon[41183]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:45.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:44 smithi181 ceph-mon[41183]: osdmap e103: 8 total, 8 up, 8 in 2024-03-29T04:53:45.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:44 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1578338328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:45.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:44 smithi181 ceph-mon[29765]: pgmap v140: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:45.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:44 smithi181 ceph-mon[29765]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:53:45.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:44 smithi181 ceph-mon[29765]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:45.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:44 smithi181 ceph-mon[29765]: osdmap e103: 8 total, 8 up, 8 in 2024-03-29T04:53:45.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:44 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1578338328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:45.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:44 smithi187 ceph-mon[33595]: pgmap v140: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:45.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:44 smithi187 ceph-mon[33595]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:53:45.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:44 smithi187 ceph-mon[33595]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:45.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:44 smithi187 ceph-mon[33595]: osdmap e103: 8 total, 8 up, 8 in 2024-03-29T04:53:45.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:44 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1578338328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:45.954 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroringWatcher.ModeUpdated (2045 ms) 2024-03-29T04:53:45.954 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroringWatcher.ImageStatusUpdated 2024-03-29T04:53:45.974 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroringWatcher.ImageStatusUpdated (21 ms) 2024-03-29T04:53:46.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:45 smithi181 ceph-mon[29765]: osdmap e104: 8 total, 8 up, 8 in 2024-03-29T04:53:46.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:45 smithi181 ceph-mon[41183]: osdmap e104: 8 total, 8 up, 8 in 2024-03-29T04:53:46.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:45 smithi187 ceph-mon[33595]: osdmap e104: 8 total, 8 up, 8 in 2024-03-29T04:53:46.926 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 2 tests from TestMirroringWatcher (2066 ms total) 2024-03-29T04:53:46.926 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T04:53:46.926 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 57 tests from TestMigration 2024-03-29T04:53:47.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:46 smithi181 ceph-mon[29765]: pgmap v143: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:47.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:46 smithi181 ceph-mon[29765]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:53:47.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:46 smithi181 ceph-mon[29765]: Cluster is now healthy 2024-03-29T04:53:47.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:46 smithi181 ceph-mon[29765]: osdmap e105: 8 total, 8 up, 8 in 2024-03-29T04:53:47.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:46 smithi181 ceph-mon[41183]: pgmap v143: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:47.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:46 smithi181 ceph-mon[41183]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:53:47.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:46 smithi181 ceph-mon[41183]: Cluster is now healthy 2024-03-29T04:53:47.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:46 smithi181 ceph-mon[41183]: osdmap e105: 8 total, 8 up, 8 in 2024-03-29T04:53:47.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:46 smithi187 ceph-mon[33595]: pgmap v143: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:47.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:46 smithi187 ceph-mon[33595]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:53:47.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:46 smithi187 ceph-mon[33595]: Cluster is now healthy 2024-03-29T04:53:47.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:46 smithi187 ceph-mon[33595]: osdmap e105: 8 total, 8 up, 8 in 2024-03-29T04:53:48.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:47 smithi181 ceph-mon[29765]: osdmap e106: 8 total, 8 up, 8 in 2024-03-29T04:53:48.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:47 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1134591463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:48.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:47 smithi181 ceph-mon[41183]: osdmap e106: 8 total, 8 up, 8 in 2024-03-29T04:53:48.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:47 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1134591463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:48.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:47 smithi187 ceph-mon[33595]: osdmap e106: 8 total, 8 up, 8 in 2024-03-29T04:53:48.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:47 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1134591463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:48.881 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:53:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:53:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:53:49.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:48 smithi181 ceph-mon[29765]: pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:49.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:48 smithi181 ceph-mon[29765]: osdmap e107: 8 total, 8 up, 8 in 2024-03-29T04:53:49.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:48 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2071555851' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:49.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:48 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:49.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:48 smithi181 ceph-mon[41183]: pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:49.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:48 smithi181 ceph-mon[41183]: osdmap e107: 8 total, 8 up, 8 in 2024-03-29T04:53:49.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:48 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2071555851' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:49.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:48 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:49.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:48 smithi187 ceph-mon[33595]: pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:49.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:48 smithi187 ceph-mon[33595]: osdmap e107: 8 total, 8 up, 8 in 2024-03-29T04:53:49.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:48 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2071555851' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:53:49.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:48 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:49.942 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.Empty 2024-03-29T04:53:50.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:49 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2071555851' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:50.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:49 smithi181 ceph-mon[29765]: osdmap e108: 8 total, 8 up, 8 in 2024-03-29T04:53:50.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:49 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1971841417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:50.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:49 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2071555851' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:50.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:49 smithi181 ceph-mon[41183]: osdmap e108: 8 total, 8 up, 8 in 2024-03-29T04:53:50.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:49 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1971841417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:50.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:49 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2071555851' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:53:50.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:49 smithi187 ceph-mon[33595]: osdmap e108: 8 total, 8 up, 8 in 2024-03-29T04:53:50.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:49 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1971841417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:51.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:50 smithi187 ceph-mon[33595]: pgmap v149: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:51.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:50 smithi187 ceph-mon[33595]: osdmap e109: 8 total, 8 up, 8 in 2024-03-29T04:53:51.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:50 smithi181 ceph-mon[29765]: pgmap v149: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:51.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:50 smithi181 ceph-mon[29765]: osdmap e109: 8 total, 8 up, 8 in 2024-03-29T04:53:51.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:50 smithi181 ceph-mon[41183]: pgmap v149: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:51.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:50 smithi181 ceph-mon[41183]: osdmap e109: 8 total, 8 up, 8 in 2024-03-29T04:53:51.997 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:52.007 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:52.148 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:53:52.219 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:53:52.300 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.Empty (2359 ms) 2024-03-29T04:53:52.300 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.OtherName 2024-03-29T04:53:52.327 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:52.338 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:52.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:51 smithi187 ceph-mon[33595]: osdmap e110: 8 total, 8 up, 8 in 2024-03-29T04:53:52.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:51 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/4230241796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:52.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:51 smithi181 ceph-mon[29765]: osdmap e110: 8 total, 8 up, 8 in 2024-03-29T04:53:52.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:51 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/4230241796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:52.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:51 smithi181 ceph-mon[41183]: osdmap e110: 8 total, 8 up, 8 in 2024-03-29T04:53:52.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:51 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/4230241796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:52.466 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:53:52.530 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:53:52.610 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.OtherName (309 ms) 2024-03-29T04:53:52.610 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.OtherPool 2024-03-29T04:53:52.636 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:52.647 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:53.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:52 smithi187 ceph-mon[33595]: pgmap v152: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:53.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:52 smithi187 ceph-mon[33595]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:53:53.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:52 smithi187 ceph-mon[33595]: osdmap e111: 8 total, 8 up, 8 in 2024-03-29T04:53:53.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:52 smithi181 ceph-mon[29765]: pgmap v152: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:53.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:52 smithi181 ceph-mon[29765]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:53:53.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:52 smithi181 ceph-mon[29765]: osdmap e111: 8 total, 8 up, 8 in 2024-03-29T04:53:53.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:52 smithi181 ceph-mon[41183]: pgmap v152: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:53.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:52 smithi181 ceph-mon[41183]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:53:53.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:52 smithi181 ceph-mon[41183]: osdmap e111: 8 total, 8 up, 8 in 2024-03-29T04:53:54.064 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:53:54.122 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:53:54.198 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.OtherPool (1588 ms) 2024-03-29T04:53:54.198 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.OtherNamespace 2024-03-29T04:53:54.226 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:54.237 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:54.364 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:53:54.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:53 smithi187 ceph-mon[33595]: osdmap e112: 8 total, 8 up, 8 in 2024-03-29T04:53:54.428 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:53:54.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:53 smithi181 ceph-mon[41183]: osdmap e112: 8 total, 8 up, 8 in 2024-03-29T04:53:54.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:53 smithi181 ceph-mon[29765]: osdmap e112: 8 total, 8 up, 8 in 2024-03-29T04:53:54.507 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.OtherNamespace (309 ms) 2024-03-29T04:53:54.507 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.DataPool 2024-03-29T04:53:54.534 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:54.546 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:54.676 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:53:54.741 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:53:54.818 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.DataPool (312 ms) 2024-03-29T04:53:54.819 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.AbortAfterPrepare 2024-03-29T04:53:54.846 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:54.859 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:54.951 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:53:54 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:53:54] "GET /metrics HTTP/1.1" 200 242318 "" "Prometheus/2.43.0" 2024-03-29T04:53:54.989 INFO:tasks.workunit.client.0.smithi181.stdout:migration_abort 2024-03-29T04:53:55.081 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.AbortAfterPrepare (263 ms) 2024-03-29T04:53:55.082 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.AbortAfterFailedPrepare 2024-03-29T04:53:55.110 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:55.120 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:55.216 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.AbortAfterFailedPrepare (135 ms) 2024-03-29T04:53:55.217 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.AbortAfterExecute 2024-03-29T04:53:55.244 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:55.257 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:55.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:54 smithi187 ceph-mon[33595]: pgmap v155: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:55.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:54 smithi187 ceph-mon[33595]: osdmap e113: 8 total, 8 up, 8 in 2024-03-29T04:53:55.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:54 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1300974807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:55.392 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:53:55.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:54 smithi181 ceph-mon[29765]: pgmap v155: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:55.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:54 smithi181 ceph-mon[29765]: osdmap e113: 8 total, 8 up, 8 in 2024-03-29T04:53:55.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:54 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1300974807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:55.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:54 smithi181 ceph-mon[41183]: pgmap v155: 137 pgs: 32 unknown, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:53:55.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:54 smithi181 ceph-mon[41183]: osdmap e113: 8 total, 8 up, 8 in 2024-03-29T04:53:55.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:54 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1300974807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:55.455 INFO:tasks.workunit.client.0.smithi181.stdout:migration_abort 2024-03-29T04:53:55.550 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.AbortAfterExecute (334 ms) 2024-03-29T04:53:55.550 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.OtherPoolAbortAfterExecute 2024-03-29T04:53:55.576 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:55.586 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:55.707 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:53:55.768 INFO:tasks.workunit.client.0.smithi181.stdout:migration_abort 2024-03-29T04:53:55.861 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.OtherPoolAbortAfterExecute (311 ms) 2024-03-29T04:53:55.861 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.OtherNamespaceAbortAfterExecute 2024-03-29T04:53:55.891 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:55.903 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:56.029 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:53:56.092 INFO:tasks.workunit.client.0.smithi181.stdout:migration_abort 2024-03-29T04:53:56.197 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.OtherNamespaceAbortAfterExecute (336 ms) 2024-03-29T04:53:56.197 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.MirroringSamePool 2024-03-29T04:53:56.225 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:56.234 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:56.240 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.MirroringSamePool (43 ms) 2024-03-29T04:53:56.240 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.MirroringAbort 2024-03-29T04:53:56.273 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:56.285 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:56.290 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.MirroringAbort (50 ms) 2024-03-29T04:53:56.290 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.MirroringOtherPoolDisabled 2024-03-29T04:53:56.319 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:56.329 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:56.336 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.MirroringOtherPoolDisabled (46 ms) 2024-03-29T04:53:56.336 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.MirroringOtherPoolEnabled 2024-03-29T04:53:56.364 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:56.374 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:56.381 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.MirroringOtherPoolEnabled (45 ms) 2024-03-29T04:53:56.381 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.MirroringPool 2024-03-29T04:53:56.409 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:56.419 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:53:56.425 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.MirroringPool (44 ms) 2024-03-29T04:53:56.426 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.Group 2024-03-29T04:53:56.453 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:56.472 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:56.603 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:53:56.665 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:53:56.753 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.Group (328 ms) 2024-03-29T04:53:56.753 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.GroupAbort 2024-03-29T04:53:56.780 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:56.798 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:56.926 INFO:tasks.workunit.client.0.smithi181.stdout:migration_abort 2024-03-29T04:53:57.037 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.GroupAbort (284 ms) 2024-03-29T04:53:57.037 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.NoSnaps 2024-03-29T04:53:57.066 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:57.076 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:53:57.138 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 16777218~8388609 2024-03-29T04:53:57.198 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:57.378 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:53:57.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:56 smithi187 ceph-mon[33595]: pgmap v157: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 595 KiB/s rd, 35 KiB/s wr, 750 op/s 2024-03-29T04:53:57.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:56 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/436319145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:57.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:56 smithi181 ceph-mon[29765]: pgmap v157: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 595 KiB/s rd, 35 KiB/s wr, 750 op/s 2024-03-29T04:53:57.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:56 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/436319145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:57.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:56 smithi181 ceph-mon[41183]: pgmap v157: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 595 KiB/s rd, 35 KiB/s wr, 750 op/s 2024-03-29T04:53:57.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:56 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/436319145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:57.502 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:53:57.619 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.NoSnaps (582 ms) 2024-03-29T04:53:57.619 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.NoSnapsOtherPool 2024-03-29T04:53:57.647 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:57.657 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:53:57.718 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 16777218~8388609 2024-03-29T04:53:57.764 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:53:57.768 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 16777218~8388609 2024-03-29T04:53:57.827 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:57.994 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:53:58.110 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:53:58.234 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.NoSnapsOtherPool (615 ms) 2024-03-29T04:53:58.234 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.NoSnapsDataPool 2024-03-29T04:53:58.261 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:58.271 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:53:58.324 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 16777218~8388609 2024-03-29T04:53:58.369 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:58.544 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:53:58.650 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:53:58.768 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.NoSnapsDataPool (534 ms) 2024-03-29T04:53:58.769 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.NoSnapsShrinkAfterPrepare 2024-03-29T04:53:58.799 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:58.811 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:53:58.874 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 16777218~8388609 2024-03-29T04:53:58.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:58 smithi187 ceph-mon[33595]: osdmap e114: 8 total, 8 up, 8 in 2024-03-29T04:53:58.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:58 smithi187 ceph-mon[33595]: pgmap v159: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 491 KiB/s rd, 29 KiB/s wr, 619 op/s 2024-03-29T04:53:58.881 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:53:58 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[35686]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:53:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:53:58.930 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:58.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:58 smithi181 ceph-mon[29765]: osdmap e114: 8 total, 8 up, 8 in 2024-03-29T04:53:58.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:58 smithi181 ceph-mon[29765]: pgmap v159: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 491 KiB/s rd, 29 KiB/s wr, 619 op/s 2024-03-29T04:53:58.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:58 smithi181 ceph-mon[41183]: osdmap e114: 8 total, 8 up, 8 in 2024-03-29T04:53:58.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:58 smithi181 ceph-mon[41183]: pgmap v159: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 491 KiB/s rd, 29 KiB/s wr, 619 op/s 2024-03-29T04:53:59.125 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 41943040 2024-03-29T04:53:59.134 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:53:59.252 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:53:59.354 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.NoSnapsShrinkAfterPrepare (586 ms) 2024-03-29T04:53:59.354 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroBeforePrepare 2024-03-29T04:53:59.380 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:59.390 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:53:59.443 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 16777218~8388609 2024-03-29T04:53:59.503 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 0 2024-03-29T04:53:59.516 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:59.610 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:53:59.677 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:53:59.721 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroBeforePrepare (367 ms) 2024-03-29T04:53:59.722 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroAfterPrepare 2024-03-29T04:53:59.749 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:53:59.760 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:53:59.822 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 16777218~8388609 2024-03-29T04:53:59.874 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:53:59.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:53:59 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/889865258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:59.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:53:59 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/889865258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:53:59.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:53:59 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/889865258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:00.051 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 0 2024-03-29T04:54:00.063 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:54:00.126 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:54:00.176 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroAfterPrepare (454 ms) 2024-03-29T04:54:00.176 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.NoSnapsExpandAfterPrepare 2024-03-29T04:54:00.206 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:54:00.217 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:54:00.279 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 16777218~8388609 2024-03-29T04:54:00.329 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:54:00.514 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 167772160 2024-03-29T04:54:00.522 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:54:00.633 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:54:00.765 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.NoSnapsExpandAfterPrepare (590 ms) 2024-03-29T04:54:00.765 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.NoSnapsSnapAfterPrepare 2024-03-29T04:54:00.794 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:54:00.804 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:54:00.858 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 16777218~8388609 2024-03-29T04:54:00.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:00 smithi187 ceph-mon[33595]: pgmap v160: 137 pgs: 137 active+clean; 129 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 39 MiB/s wr, 2.22k op/s 2024-03-29T04:54:00.908 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:54:00.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:00 smithi181 ceph-mon[29765]: pgmap v160: 137 pgs: 137 active+clean; 129 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 39 MiB/s wr, 2.22k op/s 2024-03-29T04:54:00.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:00 smithi181 ceph-mon[41183]: pgmap v160: 137 pgs: 137 active+clean; 129 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 39 MiB/s wr, 2.22k op/s 2024-03-29T04:54:01.081 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: after_prepare_snap 2024-03-29T04:54:01.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:01 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/345815844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:01.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:01 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/345815844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:01.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:01 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/345815844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:02.490 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 41943040 2024-03-29T04:54:02.512 INFO:tasks.workunit.client.0.smithi181.stdout:write: * 0~1000 2024-03-29T04:54:02.518 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:54:02.631 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:54:02.774 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.NoSnapsSnapAfterPrepare (2008 ms) 2024-03-29T04:54:02.774 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.Snaps 2024-03-29T04:54:02.803 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:54:02.813 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:54:02.867 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:54:02.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:02 smithi187 ceph-mon[33595]: osdmap e115: 8 total, 8 up, 8 in 2024-03-29T04:54:02.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:02 smithi187 ceph-mon[33595]: pgmap v162: 137 pgs: 137 active+clean; 129 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s rd, 33 MiB/s wr, 1.93k op/s 2024-03-29T04:54:02.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:02 smithi181 ceph-mon[41183]: osdmap e115: 8 total, 8 up, 8 in 2024-03-29T04:54:02.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:02 smithi181 ceph-mon[41183]: pgmap v162: 137 pgs: 137 active+clean; 129 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s rd, 33 MiB/s wr, 1.93k op/s 2024-03-29T04:54:02.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:02 smithi181 ceph-mon[29765]: osdmap e115: 8 total, 8 up, 8 in 2024-03-29T04:54:02.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:02 smithi181 ceph-mon[29765]: pgmap v162: 137 pgs: 137 active+clean; 129 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s rd, 33 MiB/s wr, 1.93k op/s 2024-03-29T04:54:03.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:03 smithi187 ceph-mon[33595]: osdmap e116: 8 total, 8 up, 8 in 2024-03-29T04:54:03.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:03 smithi181 ceph-mon[41183]: osdmap e116: 8 total, 8 up, 8 in 2024-03-29T04:54:03.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:03 smithi181 ceph-mon[29765]: osdmap e116: 8 total, 8 up, 8 in 2024-03-29T04:54:04.506 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 8388609~8388609 2024-03-29T04:54:04.511 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:04 smithi187 ceph-mon[33595]: osdmap e117: 8 total, 8 up, 8 in 2024-03-29T04:54:04.511 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:04 smithi187 ceph-mon[33595]: pgmap v165: 137 pgs: 137 active+clean; 129 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 55 MiB/s rd, 43 MiB/s wr, 1.88k op/s 2024-03-29T04:54:04.512 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:04.512 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:04.512 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:04 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:04.512 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:04 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/532492246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:04.517 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 0~1000 2024-03-29T04:54:04.517 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 1010~1000 2024-03-29T04:54:04.517 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap2 2024-03-29T04:54:04.951 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:04 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:54:04] "GET /metrics HTTP/1.1" 200 245508 "" "Prometheus/2.43.0" 2024-03-29T04:54:04.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:04 smithi181 ceph-mon[29765]: osdmap e117: 8 total, 8 up, 8 in 2024-03-29T04:54:04.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:04 smithi181 ceph-mon[29765]: pgmap v165: 137 pgs: 137 active+clean; 129 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 55 MiB/s rd, 43 MiB/s wr, 1.88k op/s 2024-03-29T04:54:04.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:04.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:04.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:04 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:04.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:04 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/532492246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:04.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:04 smithi181 ceph-mon[41183]: osdmap e117: 8 total, 8 up, 8 in 2024-03-29T04:54:04.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:04 smithi181 ceph-mon[41183]: pgmap v165: 137 pgs: 137 active+clean; 129 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 55 MiB/s rd, 43 MiB/s wr, 1.88k op/s 2024-03-29T04:54:04.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:04.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:04.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:04 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:04.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:04 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/532492246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:05.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:05 smithi187 ceph-mon[33595]: osdmap e118: 8 total, 8 up, 8 in 2024-03-29T04:54:05.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:05 smithi181 ceph-mon[29765]: osdmap e118: 8 total, 8 up, 8 in 2024-03-29T04:54:05.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:05 smithi181 ceph-mon[41183]: osdmap e118: 8 total, 8 up, 8 in 2024-03-29T04:54:06.511 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:06 smithi187 ceph-mon[33595]: osdmap e119: 8 total, 8 up, 8 in 2024-03-29T04:54:06.511 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:06 smithi187 ceph-mon[33595]: pgmap v168: 137 pgs: 137 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 57 MiB/s rd, 40 MiB/s wr, 1.11k op/s 2024-03-29T04:54:06.511 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:06 smithi187 ceph-mon[33595]: Upgrade: Updating mgr.x 2024-03-29T04:54:06.511 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:06 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:06.511 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:06 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:54:06.511 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:06 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:54:06.529 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 8388609~1000 2024-03-29T04:54:06.534 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 16777228~1000 2024-03-29T04:54:06.534 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 167772160 2024-03-29T04:54:06.542 INFO:tasks.workunit.client.0.smithi181.stdout:write: 2 83886079~8388609 2024-03-29T04:54:06.558 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap3 2024-03-29T04:54:06.803 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:06 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:54:06.803 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:06 smithi187 ceph-mon[33595]: Deploying daemon mgr.x on smithi187 2024-03-29T04:54:06.803 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:06 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3238070370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:06.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[29765]: osdmap e119: 8 total, 8 up, 8 in 2024-03-29T04:54:06.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[29765]: pgmap v168: 137 pgs: 137 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 57 MiB/s rd, 40 MiB/s wr, 1.11k op/s 2024-03-29T04:54:06.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[29765]: Upgrade: Updating mgr.x 2024-03-29T04:54:06.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:06.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:54:06.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:54:06.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:54:06.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[29765]: Deploying daemon mgr.x on smithi187 2024-03-29T04:54:06.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3238070370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[41183]: osdmap e119: 8 total, 8 up, 8 in 2024-03-29T04:54:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[41183]: pgmap v168: 137 pgs: 137 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 57 MiB/s rd, 40 MiB/s wr, 1.11k op/s 2024-03-29T04:54:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[41183]: Upgrade: Updating mgr.x 2024-03-29T04:54:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:54:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:54:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:54:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[41183]: Deploying daemon mgr.x on smithi187 2024-03-29T04:54:06.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:06 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3238070370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:07.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:07 smithi187 ceph-mon[33595]: osdmap e120: 8 total, 8 up, 8 in 2024-03-29T04:54:07.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:07 smithi187 systemd[1]: Stopping Ceph mgr.x for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:54:07.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:07 smithi181 ceph-mon[29765]: osdmap e120: 8 total, 8 up, 8 in 2024-03-29T04:54:07.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:07 smithi181 ceph-mon[41183]: osdmap e120: 8 total, 8 up, 8 in 2024-03-29T04:54:08.286 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:08 smithi187 podman[69974]: 2024-03-29 04:54:08.004880562 +0000 UTC m=+0.113684972 container died f668da78a625b0b98ff26572a171402156e141a374d4ad282a9ff9f2cba44cc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=quincy-4213e17, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-03-29T04:54:08.536 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:54:08.555 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 83886079~1 2024-03-29T04:54:08.563 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:54:08.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:08 smithi187 ceph-mon[33595]: osdmap e121: 8 total, 8 up, 8 in 2024-03-29T04:54:08.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:08 smithi187 ceph-mon[33595]: pgmap v171: 137 pgs: 137 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 57 MiB/s rd, 40 MiB/s wr, 1.11k op/s 2024-03-29T04:54:08.631 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:08 smithi187 podman[69974]: 2024-03-29 04:54:08.489489542 +0000 UTC m=+0.598293984 container cleanup f668da78a625b0b98ff26572a171402156e141a374d4ad282a9ff9f2cba44cc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-29T04:54:08.632 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:08 smithi187 bash[69974]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x 2024-03-29T04:54:08.852 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:08 smithi181 ceph-mon[41183]: osdmap e121: 8 total, 8 up, 8 in 2024-03-29T04:54:08.852 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:08 smithi181 ceph-mon[41183]: pgmap v171: 137 pgs: 137 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 57 MiB/s rd, 40 MiB/s wr, 1.11k op/s 2024-03-29T04:54:08.853 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:08 smithi181 ceph-mon[29765]: osdmap e121: 8 total, 8 up, 8 in 2024-03-29T04:54:08.853 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:08 smithi181 ceph-mon[29765]: pgmap v171: 137 pgs: 137 active+clean; 225 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 57 MiB/s rd, 40 MiB/s wr, 1.11k op/s 2024-03-29T04:54:09.131 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:08 smithi187 podman[69993]: 2024-03-29 04:54:08.757713681 +0000 UTC m=+0.742660285 container remove f668da78a625b0b98ff26572a171402156e141a374d4ad282a9ff9f2cba44cc8 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=quincy-4213e17, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-03-29T04:54:09.131 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:08 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:54:09.131 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:08 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.x.service: Failed with result 'exit-code'. 2024-03-29T04:54:09.131 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:08 smithi187 systemd[1]: Stopped Ceph mgr.x for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:54:09.131 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:08 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.x.service: Consumed 17.413s CPU time. 2024-03-29T04:54:09.437 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:09 smithi187 systemd[1]: Starting Ceph mgr.x for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:54:09.437 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:09 smithi187 podman[70129]: 2024-03-29 04:54:09.334046762 +0000 UTC m=+0.106835836 container create eac04768500a0680bb79bd8936ea2f17a56516e37c0122105850f88811b4babf (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-03-29T04:54:09.437 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:09 smithi187 podman[70129]: 2024-03-29 04:54:09.244771161 +0000 UTC m=+0.017560259 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:54:09.780 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:09 smithi181 ceph-mon[41183]: osdmap e122: 8 total, 8 up, 8 in 2024-03-29T04:54:09.780 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:09 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3375233971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:09.780 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:09 smithi181 ceph-mon[29765]: osdmap e122: 8 total, 8 up, 8 in 2024-03-29T04:54:09.780 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:09 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3375233971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:09.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:09 smithi187 ceph-mon[33595]: osdmap e122: 8 total, 8 up, 8 in 2024-03-29T04:54:09.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:09 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3375233971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:09.881 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:09 smithi187 podman[70129]: 2024-03-29 04:54:09.556836194 +0000 UTC m=+0.329625268 container init eac04768500a0680bb79bd8936ea2f17a56516e37c0122105850f88811b4babf (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362) 2024-03-29T04:54:09.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:09 smithi187 podman[70129]: 2024-03-29 04:54:09.562286692 +0000 UTC m=+0.335075767 container start eac04768500a0680bb79bd8936ea2f17a56516e37c0122105850f88811b4babf (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326) 2024-03-29T04:54:09.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:09 smithi187 ceph-mgr[70169]: -- 172.21.15.187:0/440543495 <== mon.1 v2:172.21.15.187:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56423dbf65a0 con 0x56423dbcb000 2024-03-29T04:54:09.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:09 smithi187 bash[70129]: eac04768500a0680bb79bd8936ea2f17a56516e37c0122105850f88811b4babf 2024-03-29T04:54:09.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:09 smithi187 systemd[1]: Started Ceph mgr.x for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:54:10.541 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:10.140+0000 7f2577e09200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:54:10.541 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:10.242+0000 7f2577e09200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:54:10.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:10 smithi181 ceph-mon[29765]: osdmap e123: 8 total, 8 up, 8 in 2024-03-29T04:54:10.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:10 smithi181 ceph-mon[29765]: pgmap v174: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-03-29T04:54:10.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:10 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:10.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:10 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:10.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:10 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:54:10.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:10 smithi181 ceph-mon[41183]: osdmap e123: 8 total, 8 up, 8 in 2024-03-29T04:54:10.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:10 smithi181 ceph-mon[41183]: pgmap v174: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-03-29T04:54:10.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:10 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:10.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:10 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:10.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:10 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:54:10.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:10 smithi187 ceph-mon[33595]: osdmap e123: 8 total, 8 up, 8 in 2024-03-29T04:54:10.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:10 smithi187 ceph-mon[33595]: pgmap v174: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-03-29T04:54:10.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:10 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:10.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:10 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:10.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:10 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:54:11.232 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:10.894+0000 7f2577e09200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:54:11.232 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:10.987+0000 7f2577e09200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:54:11.233 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:11.130+0000 7f2577e09200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:54:11.554 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:11.227+0000 7f2577e09200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:54:11.554 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:11.318+0000 7f2577e09200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:54:11.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:11 smithi187 ceph-mon[33595]: osdmap e124: 8 total, 8 up, 8 in 2024-03-29T04:54:11.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:11 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/119695989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:11.881 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:11.590+0000 7f2577e09200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:54:11.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:11.705+0000 7f2577e09200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:54:11.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:11.794+0000 7f2577e09200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:54:11.948 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:54:11.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:11 smithi181 ceph-mon[29765]: osdmap e124: 8 total, 8 up, 8 in 2024-03-29T04:54:11.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:11 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/119695989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:11.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:11 smithi181 ceph-mon[41183]: osdmap e124: 8 total, 8 up, 8 in 2024-03-29T04:54:11.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:11 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/119695989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:12.086 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:54:12.381 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:11.894+0000 7f2577e09200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:54:12.381 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:12 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:12.096+0000 7f2577e09200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:54:12.701 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:12 smithi181 ceph-mon[41183]: osdmap e125: 8 total, 8 up, 8 in 2024-03-29T04:54:12.701 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:12 smithi181 ceph-mon[41183]: pgmap v177: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-03-29T04:54:12.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:12 smithi181 ceph-mon[29765]: osdmap e125: 8 total, 8 up, 8 in 2024-03-29T04:54:12.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:12 smithi181 ceph-mon[29765]: pgmap v177: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-03-29T04:54:12.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:12 smithi187 ceph-mon[33595]: osdmap e125: 8 total, 8 up, 8 in 2024-03-29T04:54:12.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:12 smithi187 ceph-mon[33595]: pgmap v177: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-03-29T04:54:13.381 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:13 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:13.104+0000 7f2577e09200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:54:13.381 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:13 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:13.236+0000 7f2577e09200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:54:13.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:13 smithi187 ceph-mon[33595]: osdmap e126: 8 total, 8 up, 8 in 2024-03-29T04:54:13.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:13 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:13.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:13 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:13.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:13 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2306803789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:13.880 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:13 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:13.581+0000 7f2577e09200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:54:13.880 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:13 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:13.674+0000 7f2577e09200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:54:13.881 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:13 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:13.874+0000 7f2577e09200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:54:13.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:13 smithi181 ceph-mon[41183]: osdmap e126: 8 total, 8 up, 8 in 2024-03-29T04:54:13.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:13 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:13.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:13 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:13.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:13 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2306803789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:13.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:13 smithi181 ceph-mon[29765]: osdmap e126: 8 total, 8 up, 8 in 2024-03-29T04:54:13.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:13 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:13.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:13 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:13.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:13 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2306803789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:14.131 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:14 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:14.068+0000 7f2577e09200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:54:14.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:14 smithi187 ceph-mon[33595]: osdmap e127: 8 total, 8 up, 8 in 2024-03-29T04:54:14.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:14 smithi187 ceph-mon[33595]: pgmap v180: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:54:14.631 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:14 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:14.361+0000 7f2577e09200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:54:14.861 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.Snaps (12087 ms) 2024-03-29T04:54:14.861 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.SnapsOtherPool 2024-03-29T04:54:14.890 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:54:14.901 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:54:14.906 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:14 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:14.641+0000 7f2577e09200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:54:14.907 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:14 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:14.901+0000 7f2577e09200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:54:14.951 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:14 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:54:14] "GET /metrics HTTP/1.1" 200 245508 "" "Prometheus/2.43.0" 2024-03-29T04:54:14.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:14 smithi181 ceph-mon[29765]: osdmap e127: 8 total, 8 up, 8 in 2024-03-29T04:54:14.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:14 smithi181 ceph-mon[29765]: pgmap v180: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:54:14.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:14 smithi181 ceph-mon[41183]: osdmap e127: 8 total, 8 up, 8 in 2024-03-29T04:54:14.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:14 smithi181 ceph-mon[41183]: pgmap v180: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:54:14.965 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:54:15.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:15 smithi187 ceph-mon[33595]: osdmap e128: 8 total, 8 up, 8 in 2024-03-29T04:54:15.881 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:15 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:15.746+0000 7f2577e09200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:54:15.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:15 smithi181 ceph-mon[29765]: osdmap e128: 8 total, 8 up, 8 in 2024-03-29T04:54:15.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:15 smithi181 ceph-mon[41183]: osdmap e128: 8 total, 8 up, 8 in 2024-03-29T04:54:16.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:16 smithi187 ceph-mon[33595]: osdmap e129: 8 total, 8 up, 8 in 2024-03-29T04:54:16.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:16 smithi187 ceph-mon[33595]: pgmap v183: 137 pgs: 137 active+clean; 249 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 6.0 MiB/s wr, 427 op/s 2024-03-29T04:54:16.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:16 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/425987687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:16.632 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:16.345+0000 7f2577e09200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:54:16.632 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:16.449+0000 7f2577e09200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:54:16.675 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 8388609~8388609 2024-03-29T04:54:16.697 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 0~1000 2024-03-29T04:54:16.697 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 1010~1000 2024-03-29T04:54:16.697 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap2 2024-03-29T04:54:16.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:16 smithi181 ceph-mon[41183]: osdmap e129: 8 total, 8 up, 8 in 2024-03-29T04:54:16.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:16 smithi181 ceph-mon[41183]: pgmap v183: 137 pgs: 137 active+clean; 249 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 6.0 MiB/s wr, 427 op/s 2024-03-29T04:54:16.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:16 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/425987687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:16.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:16 smithi181 ceph-mon[29765]: osdmap e129: 8 total, 8 up, 8 in 2024-03-29T04:54:16.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:16 smithi181 ceph-mon[29765]: pgmap v183: 137 pgs: 137 active+clean; 249 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 6.0 MiB/s wr, 427 op/s 2024-03-29T04:54:16.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:16 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/425987687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:17.131 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:16.673+0000 7f2577e09200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:54:17.131 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: [29/Mar/2024:04:54:16] ENGINE Bus STARTING 2024-03-29T04:54:17.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: CherryPy Checker: 2024-03-29T04:54:17.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: The Application mounted at '' has an empty config. 2024-03-29T04:54:17.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:17.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: [29/Mar/2024:04:54:16] ENGINE Serving on http://:::9283 2024-03-29T04:54:17.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: [29/Mar/2024:04:54:16] ENGINE Bus STARTED 2024-03-29T04:54:17.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:17 smithi187 ceph-mon[33595]: osdmap e130: 8 total, 8 up, 8 in 2024-03-29T04:54:17.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:17 smithi187 ceph-mon[33595]: Standby manager daemon x restarted 2024-03-29T04:54:17.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:17 smithi187 ceph-mon[33595]: Standby manager daemon x started 2024-03-29T04:54:17.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:17 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:54:17.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:17 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:54:17.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:17 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:54:17.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:17 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:54:17.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:17 smithi187 ceph-mon[33595]: osdmap e131: 8 total, 8 up, 8 in 2024-03-29T04:54:17.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:17 smithi181 ceph-mon[29765]: osdmap e130: 8 total, 8 up, 8 in 2024-03-29T04:54:17.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:17 smithi181 ceph-mon[29765]: Standby manager daemon x restarted 2024-03-29T04:54:17.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:17 smithi181 ceph-mon[29765]: Standby manager daemon x started 2024-03-29T04:54:17.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:17 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:54:17.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:17 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:54:17.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:17 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:54:17.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:17 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:54:17.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:17 smithi181 ceph-mon[29765]: osdmap e131: 8 total, 8 up, 8 in 2024-03-29T04:54:17.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:17 smithi181 ceph-mon[41183]: osdmap e130: 8 total, 8 up, 8 in 2024-03-29T04:54:17.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:17 smithi181 ceph-mon[41183]: Standby manager daemon x restarted 2024-03-29T04:54:17.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:17 smithi181 ceph-mon[41183]: Standby manager daemon x started 2024-03-29T04:54:17.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:17 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:54:17.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:17 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:54:17.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:17 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:54:17.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:17 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:54:17.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:17 smithi181 ceph-mon[41183]: osdmap e131: 8 total, 8 up, 8 in 2024-03-29T04:54:18.473 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 8388609~1000 2024-03-29T04:54:18.480 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 16777228~1000 2024-03-29T04:54:18.480 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 167772160 2024-03-29T04:54:18.489 INFO:tasks.workunit.client.0.smithi181.stdout:write: 2 83886079~8388609 2024-03-29T04:54:18.507 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap3 2024-03-29T04:54:18.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:18 smithi187 ceph-mon[33595]: pgmap v186: 137 pgs: 137 active+clean; 249 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 6.0 MiB/s wr, 427 op/s 2024-03-29T04:54:18.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:18 smithi187 ceph-mon[33595]: mgrmap e21: y(active, since 3m), standbys: x 2024-03-29T04:54:18.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:18 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/311359761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:18.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:18 smithi187 ceph-mon[33595]: osdmap e132: 8 total, 8 up, 8 in 2024-03-29T04:54:18.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:18 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:18.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:54:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:54:18.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:18 smithi181 ceph-mon[41183]: pgmap v186: 137 pgs: 137 active+clean; 249 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 6.0 MiB/s wr, 427 op/s 2024-03-29T04:54:18.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:18 smithi181 ceph-mon[41183]: mgrmap e21: y(active, since 3m), standbys: x 2024-03-29T04:54:18.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:18 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/311359761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:18.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:18 smithi181 ceph-mon[41183]: osdmap e132: 8 total, 8 up, 8 in 2024-03-29T04:54:18.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:18 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:18.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:18 smithi181 ceph-mon[29765]: pgmap v186: 137 pgs: 137 active+clean; 249 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 6.0 MiB/s wr, 427 op/s 2024-03-29T04:54:18.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:18 smithi181 ceph-mon[29765]: mgrmap e21: y(active, since 3m), standbys: x 2024-03-29T04:54:18.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:18 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/311359761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:18.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:18 smithi181 ceph-mon[29765]: osdmap e132: 8 total, 8 up, 8 in 2024-03-29T04:54:18.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:18 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:20.487 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:54:20.509 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 83886079~1 2024-03-29T04:54:20.517 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:54:20.523 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 16777218~8388609 2024-03-29T04:54:20.593 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:54:20.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:20 smithi187 ceph-mon[33595]: osdmap e133: 8 total, 8 up, 8 in 2024-03-29T04:54:20.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:20 smithi187 ceph-mon[33595]: pgmap v189: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 31 MiB/s rd, 12 MiB/s wr, 165 op/s 2024-03-29T04:54:20.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:20 smithi181 ceph-mon[29765]: osdmap e133: 8 total, 8 up, 8 in 2024-03-29T04:54:20.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:20 smithi181 ceph-mon[29765]: pgmap v189: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 31 MiB/s rd, 12 MiB/s wr, 165 op/s 2024-03-29T04:54:20.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:20 smithi181 ceph-mon[41183]: osdmap e133: 8 total, 8 up, 8 in 2024-03-29T04:54:20.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:20 smithi181 ceph-mon[41183]: pgmap v189: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 31 MiB/s rd, 12 MiB/s wr, 165 op/s 2024-03-29T04:54:21.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:21 smithi187 ceph-mon[33595]: osdmap e134: 8 total, 8 up, 8 in 2024-03-29T04:54:21.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:21 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1219776445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:21.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:21 smithi181 ceph-mon[41183]: osdmap e134: 8 total, 8 up, 8 in 2024-03-29T04:54:21.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:21 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1219776445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:21.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:21 smithi181 ceph-mon[29765]: osdmap e134: 8 total, 8 up, 8 in 2024-03-29T04:54:21.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:21 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1219776445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:22.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:22 smithi187 ceph-mon[33595]: osdmap e135: 8 total, 8 up, 8 in 2024-03-29T04:54:22.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:22 smithi187 ceph-mon[33595]: pgmap v192: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 31 MiB/s rd, 12 MiB/s wr, 165 op/s 2024-03-29T04:54:22.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:22 smithi181 ceph-mon[41183]: osdmap e135: 8 total, 8 up, 8 in 2024-03-29T04:54:22.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:22 smithi181 ceph-mon[41183]: pgmap v192: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 31 MiB/s rd, 12 MiB/s wr, 165 op/s 2024-03-29T04:54:22.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:22 smithi181 ceph-mon[29765]: osdmap e135: 8 total, 8 up, 8 in 2024-03-29T04:54:22.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:22 smithi181 ceph-mon[29765]: pgmap v192: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 31 MiB/s rd, 12 MiB/s wr, 165 op/s 2024-03-29T04:54:23.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:23 smithi187 ceph-mon[33595]: osdmap e136: 8 total, 8 up, 8 in 2024-03-29T04:54:23.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:23 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2339712303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:23.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:23 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:23.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:23 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:23.920 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:54:23.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:23 smithi181 ceph-mon[41183]: osdmap e136: 8 total, 8 up, 8 in 2024-03-29T04:54:23.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:23 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2339712303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:23.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:23 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:23.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:23 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:23.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:23 smithi181 ceph-mon[29765]: osdmap e136: 8 total, 8 up, 8 in 2024-03-29T04:54:23.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:23 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2339712303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:23.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:23 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:23.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:23 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:24.095 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:54:24.746 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:24 smithi187 ceph-mon[33595]: osdmap e137: 8 total, 8 up, 8 in 2024-03-29T04:54:24.746 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:24 smithi187 ceph-mon[33595]: pgmap v195: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:54:24.746 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:24 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:24.747 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:24 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:24.951 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:24 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:54:24] "GET /metrics HTTP/1.1" 200 245601 "" "Prometheus/2.43.0" 2024-03-29T04:54:24.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:24 smithi181 ceph-mon[41183]: osdmap e137: 8 total, 8 up, 8 in 2024-03-29T04:54:24.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:24 smithi181 ceph-mon[41183]: pgmap v195: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:54:24.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:24 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:24.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:24 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:24.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:24 smithi181 ceph-mon[29765]: osdmap e137: 8 total, 8 up, 8 in 2024-03-29T04:54:24.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:24 smithi181 ceph-mon[29765]: pgmap v195: 137 pgs: 137 active+clean; 305 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:54:24.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:24 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:24.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:24 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:25.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:25 smithi187 ceph-mon[33595]: osdmap e138: 8 total, 8 up, 8 in 2024-03-29T04:54:25.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:25 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/4260835783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:25.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:25 smithi181 ceph-mon[41183]: osdmap e138: 8 total, 8 up, 8 in 2024-03-29T04:54:25.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:25 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/4260835783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:25.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:25 smithi181 ceph-mon[29765]: osdmap e138: 8 total, 8 up, 8 in 2024-03-29T04:54:25.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:25 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/4260835783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:26.796 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:26 smithi187 ceph-mon[33595]: osdmap e139: 8 total, 8 up, 8 in 2024-03-29T04:54:26.796 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:26 smithi187 ceph-mon[33595]: pgmap v198: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 18 MiB/s wr, 496 op/s 2024-03-29T04:54:26.809 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.SnapsOtherPool (11949 ms) 2024-03-29T04:54:26.809 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.SnapsDataPool 2024-03-29T04:54:26.840 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:54:26.849 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:54:26.914 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:54:26.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:26 smithi181 ceph-mon[41183]: osdmap e139: 8 total, 8 up, 8 in 2024-03-29T04:54:26.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:26 smithi181 ceph-mon[41183]: pgmap v198: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 18 MiB/s wr, 496 op/s 2024-03-29T04:54:26.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:26 smithi181 ceph-mon[29765]: osdmap e139: 8 total, 8 up, 8 in 2024-03-29T04:54:26.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:26 smithi181 ceph-mon[29765]: pgmap v198: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 18 MiB/s wr, 496 op/s 2024-03-29T04:54:27.775 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:27 smithi181 ceph-mon[29765]: osdmap e140: 8 total, 8 up, 8 in 2024-03-29T04:54:27.775 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:27 smithi181 ceph-mon[29765]: osdmap e141: 8 total, 8 up, 8 in 2024-03-29T04:54:27.776 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:27 smithi181 ceph-mon[41183]: osdmap e140: 8 total, 8 up, 8 in 2024-03-29T04:54:27.776 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:27 smithi181 ceph-mon[41183]: osdmap e141: 8 total, 8 up, 8 in 2024-03-29T04:54:27.783 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-mon[33595]: osdmap e140: 8 total, 8 up, 8 in 2024-03-29T04:54:27.783 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-mon[33595]: osdmap e141: 8 total, 8 up, 8 in 2024-03-29T04:54:27.785 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 8388609~8388609 2024-03-29T04:54:27.796 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 0~1000 2024-03-29T04:54:27.796 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 1010~1000 2024-03-29T04:54:27.796 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap2 2024-03-29T04:54:28.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: [29/Mar/2024:04:54:27] ENGINE Bus STOPPING 2024-03-29T04:54:28.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: [29/Mar/2024:04:54:27] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:54:28.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: [29/Mar/2024:04:54:27] ENGINE Bus STOPPED 2024-03-29T04:54:28.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: [29/Mar/2024:04:54:27] ENGINE Bus STARTING 2024-03-29T04:54:28.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:27.911+0000 7f24d4a9a700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:54:28.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:27.911+0000 7f24d4a9a700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:54:28.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:27.911+0000 7f24d4a9a700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:54:28.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:27.911+0000 7f24d4a9a700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:54:28.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:27.911+0000 7f24d4a9a700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:54:28.133 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:27.911+0000 7f24d8aa2700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:54:28.133 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:27.911+0000 7f24d8aa2700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:54:28.133 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:27.911+0000 7f24d8aa2700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:54:28.133 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:27.911+0000 7f24d8aa2700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:54:28.133 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:54:27.911+0000 7f24d8aa2700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:54:28.133 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: [29/Mar/2024:04:54:27] ENGINE Serving on http://:::9283 2024-03-29T04:54:28.133 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: [29/Mar/2024:04:54:27] ENGINE Bus STARTED 2024-03-29T04:54:28.201 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:27 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:27.768+0000 7f74e6986700 -1 mgr handle_mgr_map I was active but no longer am 2024-03-29T04:54:28.201 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:27 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ignoring --setuser ceph since I am not root 2024-03-29T04:54:28.201 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:27 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ignoring --setgroup ceph since I am not root 2024-03-29T04:54:28.201 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:27 smithi181 ceph-mgr[30042]: -- 172.21.15.181:0/2098799247 <== mon.0 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f13cda49c0 con 0x55f13cd7a800 2024-03-29T04:54:28.645 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:28.348+0000 7f397a8221c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:54:28.645 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:28.441+0000 7f397a8221c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:54:28.881 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:54:28] "GET /metrics HTTP/1.1" 200 35632 "" "Prometheus/2.43.0" 2024-03-29T04:54:28.882 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:54:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"172.21.15.181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:54:28.882 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:54:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephPGImbalance index=15 msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on (job) group_left () avg by (job) (ceph_osd_numpg\n > 0)) / on (job) group_left () avg by (job) (ceph_osd_numpg > 0)) * on (ceph_daemon)\n group_left (hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"172.21.15.181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:54:28.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:28.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:28.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:54:28.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:54:28.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:28.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:54:28.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-29T04:54:28.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: osdmap e142: 8 total, 8 up, 8 in 2024-03-29T04:54:28.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-03-29T04:54:28.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: mgrmap e22: x(active, starting, since 0.00922018s) 2024-03-29T04:54:28.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:54:28.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:54:28.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:54:28.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi187.vcsnqx"}]: dispatch 2024-03-29T04:54:28.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.pflrwm"}]: dispatch 2024-03-29T04:54:28.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:54:28.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:54:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:54:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:54:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:54:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:54:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:54:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:54:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:54:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:54:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:54:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:54:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: Manager daemon x is now available 2024-03-29T04:54:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:54:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/4239413453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:54:28.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:54:28.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:54:28.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:54:28.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: Deploying cephadm binary to smithi181 2024-03-29T04:54:28.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:28 smithi187 ceph-mon[33595]: Deploying cephadm binary to smithi187 2024-03-29T04:54:28.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:28.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:28.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:54:28.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:54:28.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:28.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:54:28.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-29T04:54:28.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: osdmap e142: 8 total, 8 up, 8 in 2024-03-29T04:54:28.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-03-29T04:54:28.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: mgrmap e22: x(active, starting, since 0.00922018s) 2024-03-29T04:54:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:54:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:54:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:54:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi187.vcsnqx"}]: dispatch 2024-03-29T04:54:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.pflrwm"}]: dispatch 2024-03-29T04:54:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:54:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:54:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:54:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:54:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:54:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:54:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:54:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:54:28.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:54:28.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:54:28.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:54:28.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:54:28.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: Manager daemon x is now available 2024-03-29T04:54:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:54:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:54:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' 2024-03-29T04:54:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:54:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-29T04:54:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: osdmap e142: 8 total, 8 up, 8 in 2024-03-29T04:54:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.14448 172.21.15.181:0/281088741' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-03-29T04:54:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: mgrmap e22: x(active, starting, since 0.00922018s) 2024-03-29T04:54:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:54:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:54:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:54:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi187.vcsnqx"}]: dispatch 2024-03-29T04:54:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.pflrwm"}]: dispatch 2024-03-29T04:54:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:54:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:54:28.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:54:28.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:54:28.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:54:28.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:54:28.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:54:28.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:54:28.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:54:28.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:54:28.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:54:28.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:54:28.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: Manager daemon x is now available 2024-03-29T04:54:28.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:54:28.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/4239413453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:28.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:28.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:54:28.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:54:28.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:54:28.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:54:28.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: Deploying cephadm binary to smithi181 2024-03-29T04:54:28.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[29765]: Deploying cephadm binary to smithi187 2024-03-29T04:54:28.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:54:28.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/4239413453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:28.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:28.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:54:28.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:54:28.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:54:28.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:54:28.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: Deploying cephadm binary to smithi181 2024-03-29T04:54:28.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:28 smithi181 ceph-mon[41183]: Deploying cephadm binary to smithi187 2024-03-29T04:54:29.393 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:29.069+0000 7f397a8221c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:54:29.393 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:29.156+0000 7f397a8221c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:54:29.393 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:29.238+0000 7f397a8221c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:54:29.701 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:29.387+0000 7f397a8221c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:54:29.701 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:29.466+0000 7f397a8221c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:54:29.701 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:29.633+0000 7f397a8221c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:54:29.838 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 8388609~1000 2024-03-29T04:54:29.846 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 16777228~1000 2024-03-29T04:54:29.846 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 167772160 2024-03-29T04:54:29.854 INFO:tasks.workunit.client.0.smithi181.stdout:write: 2 83886079~8388609 2024-03-29T04:54:29.874 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap3 2024-03-29T04:54:30.109 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-mon[29765]: mgrmap e23: x(active, since 1.01087s) 2024-03-29T04:54:30.109 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-mon[29765]: osdmap e143: 8 total, 8 up, 8 in 2024-03-29T04:54:30.109 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-mon[29765]: pgmap v4: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 196 GiB/s rd, 103 GiB/s wr, 4.40M op/s; 319 MiB/s, 1.13k objects/s recovering 2024-03-29T04:54:30.109 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:30.109 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:30.110 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:29.816+0000 7f397a8221c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:54:30.110 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:29.918+0000 7f397a8221c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:54:30.110 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:30.015+0000 7f397a8221c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:54:30.110 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-mon[41183]: mgrmap e23: x(active, since 1.01087s) 2024-03-29T04:54:30.110 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-mon[41183]: osdmap e143: 8 total, 8 up, 8 in 2024-03-29T04:54:30.110 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-mon[41183]: pgmap v4: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 196 GiB/s rd, 103 GiB/s wr, 4.40M op/s; 319 MiB/s, 1.13k objects/s recovering 2024-03-29T04:54:30.110 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:30.110 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:29 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:30.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:29 smithi187 ceph-mon[33595]: mgrmap e23: x(active, since 1.01087s) 2024-03-29T04:54:30.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:29 smithi187 ceph-mon[33595]: osdmap e143: 8 total, 8 up, 8 in 2024-03-29T04:54:30.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:29 smithi187 ceph-mon[33595]: pgmap v4: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 196 GiB/s rd, 103 GiB/s wr, 4.40M op/s; 319 MiB/s, 1.13k objects/s recovering 2024-03-29T04:54:30.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:29 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:30.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:29 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:30.451 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:30.103+0000 7f397a8221c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:54:31.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:30 smithi187 ceph-mon[33595]: [29/Mar/2024:04:54:29] ENGINE Bus STARTING 2024-03-29T04:54:31.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:30 smithi187 ceph-mon[33595]: pgmap v5: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.0 GiB/s rd, 564 MiB/s wr, 23.46k op/s; 1.7 MiB/s, 6 objects/s recovering 2024-03-29T04:54:31.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:30 smithi187 ceph-mon[33595]: osdmap e144: 8 total, 8 up, 8 in 2024-03-29T04:54:31.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:30 smithi187 ceph-mon[33595]: [29/Mar/2024:04:54:29] ENGINE Serving on https://172.21.15.187:7150 2024-03-29T04:54:31.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:30 smithi187 ceph-mon[33595]: [29/Mar/2024:04:54:29] ENGINE Serving on http://172.21.15.187:8765 2024-03-29T04:54:31.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:30 smithi187 ceph-mon[33595]: [29/Mar/2024:04:54:29] ENGINE Bus STARTED 2024-03-29T04:54:31.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:30 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3842146704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:31.158 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:30 smithi181 ceph-mon[29765]: [29/Mar/2024:04:54:29] ENGINE Bus STARTING 2024-03-29T04:54:31.158 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:30 smithi181 ceph-mon[29765]: pgmap v5: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.0 GiB/s rd, 564 MiB/s wr, 23.46k op/s; 1.7 MiB/s, 6 objects/s recovering 2024-03-29T04:54:31.158 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:30 smithi181 ceph-mon[29765]: osdmap e144: 8 total, 8 up, 8 in 2024-03-29T04:54:31.158 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:30 smithi181 ceph-mon[29765]: [29/Mar/2024:04:54:29] ENGINE Serving on https://172.21.15.187:7150 2024-03-29T04:54:31.158 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:30 smithi181 ceph-mon[29765]: [29/Mar/2024:04:54:29] ENGINE Serving on http://172.21.15.187:8765 2024-03-29T04:54:31.158 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:30 smithi181 ceph-mon[29765]: [29/Mar/2024:04:54:29] ENGINE Bus STARTED 2024-03-29T04:54:31.158 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:30 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3842146704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:31.158 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:30 smithi181 ceph-mon[41183]: [29/Mar/2024:04:54:29] ENGINE Bus STARTING 2024-03-29T04:54:31.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:30 smithi181 ceph-mon[41183]: pgmap v5: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.0 GiB/s rd, 564 MiB/s wr, 23.46k op/s; 1.7 MiB/s, 6 objects/s recovering 2024-03-29T04:54:31.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:30 smithi181 ceph-mon[41183]: osdmap e144: 8 total, 8 up, 8 in 2024-03-29T04:54:31.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:30 smithi181 ceph-mon[41183]: [29/Mar/2024:04:54:29] ENGINE Serving on https://172.21.15.187:7150 2024-03-29T04:54:31.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:30 smithi181 ceph-mon[41183]: [29/Mar/2024:04:54:29] ENGINE Serving on http://172.21.15.187:8765 2024-03-29T04:54:31.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:30 smithi181 ceph-mon[41183]: [29/Mar/2024:04:54:29] ENGINE Bus STARTED 2024-03-29T04:54:31.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:30 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3842146704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:31.451 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:31.152+0000 7f397a8221c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:54:31.451 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:31.283+0000 7f397a8221c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:54:31.794 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:31.475+0000 7f397a8221c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:54:31.794 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:31.704+0000 7f397a8221c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:54:31.805 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:54:31.826 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 83886079~1 2024-03-29T04:54:31.835 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:54:32.094 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:31.903+0000 7f397a8221c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:54:32.094 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:31 smithi181 ceph-mon[41183]: osdmap e145: 8 total, 8 up, 8 in 2024-03-29T04:54:32.094 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:31 smithi181 ceph-mon[41183]: mgrmap e24: x(active, since 3s) 2024-03-29T04:54:32.094 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:31 smithi181 ceph-mon[29765]: osdmap e145: 8 total, 8 up, 8 in 2024-03-29T04:54:32.095 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:31 smithi181 ceph-mon[29765]: mgrmap e24: x(active, since 3s) 2024-03-29T04:54:32.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:31 smithi187 ceph-mon[33595]: osdmap e145: 8 total, 8 up, 8 in 2024-03-29T04:54:32.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:31 smithi187 ceph-mon[33595]: mgrmap e24: x(active, since 3s) 2024-03-29T04:54:32.363 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:32 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:32.088+0000 7f397a8221c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:54:32.701 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:32 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:32.357+0000 7f397a8221c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:54:32.701 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:32 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:32.450+0000 7f397a8221c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:54:33.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:32 smithi187 ceph-mon[33595]: pgmap v8: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 356 MiB/s rd, 187 MiB/s wr, 7.78k op/s; 578 KiB/s, 2 objects/s recovering 2024-03-29T04:54:33.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:32 smithi187 ceph-mon[33595]: osdmap e146: 8 total, 8 up, 8 in 2024-03-29T04:54:33.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:32 smithi187 ceph-mon[33595]: osdmap e147: 8 total, 8 up, 8 in 2024-03-29T04:54:33.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:32 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2691709156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:33.201 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:32 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:32.854+0000 7f397a8221c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:54:33.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:32 smithi181 ceph-mon[41183]: pgmap v8: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 356 MiB/s rd, 187 MiB/s wr, 7.78k op/s; 578 KiB/s, 2 objects/s recovering 2024-03-29T04:54:33.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:32 smithi181 ceph-mon[41183]: osdmap e146: 8 total, 8 up, 8 in 2024-03-29T04:54:33.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:32 smithi181 ceph-mon[41183]: osdmap e147: 8 total, 8 up, 8 in 2024-03-29T04:54:33.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:32 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2691709156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:33.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:32 smithi181 ceph-mon[29765]: pgmap v8: 137 pgs: 137 active+clean; 369 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 356 MiB/s rd, 187 MiB/s wr, 7.78k op/s; 578 KiB/s, 2 objects/s recovering 2024-03-29T04:54:33.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:32 smithi181 ceph-mon[29765]: osdmap e146: 8 total, 8 up, 8 in 2024-03-29T04:54:33.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:32 smithi181 ceph-mon[29765]: osdmap e147: 8 total, 8 up, 8 in 2024-03-29T04:54:33.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:32 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2691709156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:33.951 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:33 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:33.680+0000 7f397a8221c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:54:34.328 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:33 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:33.991+0000 7f397a8221c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:54:34.328 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:34 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:34.321+0000 7f397a8221c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:54:34.592 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:34 smithi181 ceph-mon[29765]: osdmap e148: 8 total, 8 up, 8 in 2024-03-29T04:54:34.592 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:34 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:34.564+0000 7f397a8221c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:54:34.592 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:34 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:54:34] ENGINE Bus STARTING 2024-03-29T04:54:34.593 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:34 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:54:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:54:34.593 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:34 smithi181 ceph-mon[41183]: osdmap e148: 8 total, 8 up, 8 in 2024-03-29T04:54:34.820 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:34 smithi187 ceph-mon[33595]: osdmap e148: 8 total, 8 up, 8 in 2024-03-29T04:54:34.871 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:34 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: CherryPy Checker: 2024-03-29T04:54:34.872 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:34 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: The Application mounted at '' has an empty config. 2024-03-29T04:54:34.872 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:34 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: 2024-03-29T04:54:34.872 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:34 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:54:34] ENGINE Serving on http://:::9283 2024-03-29T04:54:34.872 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:34 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: [29/Mar/2024:04:54:34] ENGINE Bus STARTED 2024-03-29T04:54:34.932 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:54:35.069 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:54:35.765 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:35 smithi181 ceph-mon[29765]: pgmap v12: 137 pgs: 137 active+clean; 377 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 51 KiB/s rd, 1.0 MiB/s wr, 66 op/s 2024-03-29T04:54:35.766 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:35 smithi181 ceph-mon[29765]: osdmap e149: 8 total, 8 up, 8 in 2024-03-29T04:54:35.766 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:35 smithi181 ceph-mon[29765]: Standby manager daemon y started 2024-03-29T04:54:35.766 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:35 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.181:0/1971493704' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:54:35.766 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:35 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.181:0/1971493704' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:54:35.766 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:35 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.181:0/1971493704' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:54:35.766 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:35 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.181:0/1971493704' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:54:35.766 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:35 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2013012294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:35.766 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:35 smithi181 ceph-mon[41183]: pgmap v12: 137 pgs: 137 active+clean; 377 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 51 KiB/s rd, 1.0 MiB/s wr, 66 op/s 2024-03-29T04:54:35.766 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:35 smithi181 ceph-mon[41183]: osdmap e149: 8 total, 8 up, 8 in 2024-03-29T04:54:35.766 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:35 smithi181 ceph-mon[41183]: Standby manager daemon y started 2024-03-29T04:54:35.767 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:35 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.181:0/1971493704' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:54:35.767 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:35 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.181:0/1971493704' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:54:35.767 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:35 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.181:0/1971493704' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:54:35.767 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:35 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.181:0/1971493704' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:54:35.767 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:35 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2013012294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:35.799 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:35 smithi187 ceph-mon[33595]: pgmap v12: 137 pgs: 137 active+clean; 377 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 51 KiB/s rd, 1.0 MiB/s wr, 66 op/s 2024-03-29T04:54:35.799 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:35 smithi187 ceph-mon[33595]: osdmap e149: 8 total, 8 up, 8 in 2024-03-29T04:54:35.799 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:35 smithi187 ceph-mon[33595]: Standby manager daemon y started 2024-03-29T04:54:35.799 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:35 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.181:0/1971493704' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:54:35.799 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:35 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.181:0/1971493704' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:54:35.800 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:35 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.181:0/1971493704' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:54:35.800 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:35 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.181:0/1971493704' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:54:35.800 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:35 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2013012294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:36.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:36 smithi187 ceph-mon[33595]: osdmap e150: 8 total, 8 up, 8 in 2024-03-29T04:54:36.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:36 smithi187 ceph-mon[33595]: mgrmap e25: x(active, since 7s), standbys: y 2024-03-29T04:54:36.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:36 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:54:36.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:36 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:36.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:36 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:36.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:36 smithi181 ceph-mon[41183]: osdmap e150: 8 total, 8 up, 8 in 2024-03-29T04:54:36.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:36 smithi181 ceph-mon[41183]: mgrmap e25: x(active, since 7s), standbys: y 2024-03-29T04:54:36.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:36 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:54:36.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:36 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:36.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:36 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:36.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:36 smithi181 ceph-mon[29765]: osdmap e150: 8 total, 8 up, 8 in 2024-03-29T04:54:36.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:36 smithi181 ceph-mon[29765]: mgrmap e25: x(active, since 7s), standbys: y 2024-03-29T04:54:36.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:36 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:54:36.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:36 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:36.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:36 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:37.811 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.SnapsDataPool (11002 ms) 2024-03-29T04:54:37.812 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.SnapsShrinkAfterPrepare 2024-03-29T04:54:37.842 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:54:37.851 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:54:37.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:37 smithi187 ceph-mon[33595]: pgmap v15: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 172 KiB/s rd, 4.0 MiB/s wr, 201 op/s 2024-03-29T04:54:37.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:37 smithi187 ceph-mon[33595]: osdmap e151: 8 total, 8 up, 8 in 2024-03-29T04:54:37.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:37 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:37.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:37 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:37.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:37 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:37 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:37 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:37 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:37 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:37 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:37 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:37 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:37 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:37.910 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:54:37.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[29765]: pgmap v15: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 172 KiB/s rd, 4.0 MiB/s wr, 201 op/s 2024-03-29T04:54:37.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[29765]: osdmap e151: 8 total, 8 up, 8 in 2024-03-29T04:54:37.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:37.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:37.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:37.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[41183]: pgmap v15: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 172 KiB/s rd, 4.0 MiB/s wr, 201 op/s 2024-03-29T04:54:37.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[41183]: osdmap e151: 8 total, 8 up, 8 in 2024-03-29T04:54:37.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:37.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:37.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:37.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:37 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:38.853 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:38 smithi181 ceph-mon[41183]: Adjusting osd_memory_target on smithi181 to 2480M 2024-03-29T04:54:38.853 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:38 smithi181 ceph-mon[41183]: osdmap e152: 8 total, 8 up, 8 in 2024-03-29T04:54:38.853 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:38 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2209716714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:38.853 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:38 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:38.853 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:38 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:38.853 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:38 smithi181 ceph-mon[29765]: Adjusting osd_memory_target on smithi181 to 2480M 2024-03-29T04:54:38.853 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:38 smithi181 ceph-mon[29765]: osdmap e152: 8 total, 8 up, 8 in 2024-03-29T04:54:38.853 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:38 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2209716714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:38.854 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:38 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:38.854 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:38 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:38.881 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:38 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:54:38] "GET /metrics HTTP/1.1" 200 35632 "" "Prometheus/2.43.0" 2024-03-29T04:54:38.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:38 smithi187 ceph-mon[33595]: Adjusting osd_memory_target on smithi181 to 2480M 2024-03-29T04:54:38.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:38 smithi187 ceph-mon[33595]: osdmap e152: 8 total, 8 up, 8 in 2024-03-29T04:54:38.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:38 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2209716714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:38.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:38 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:38.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:38 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:39.620 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:39 smithi187 ceph-mon[33595]: pgmap v18: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 3.0 MiB/s wr, 134 op/s 2024-03-29T04:54:39.620 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:39 smithi187 ceph-mon[33595]: osdmap e153: 8 total, 8 up, 8 in 2024-03-29T04:54:39.620 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:39 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:39.620 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:39 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:39.689 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 8388609~8388609 2024-03-29T04:54:39.701 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 0~1000 2024-03-29T04:54:39.702 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 1010~1000 2024-03-29T04:54:39.702 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap2 2024-03-29T04:54:39.871 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:39 smithi181 ceph-mon[41183]: pgmap v18: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 3.0 MiB/s wr, 134 op/s 2024-03-29T04:54:39.871 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:39 smithi181 ceph-mon[41183]: osdmap e153: 8 total, 8 up, 8 in 2024-03-29T04:54:39.871 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:39 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:39.871 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:39 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:39.871 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:39 smithi181 ceph-mon[29765]: pgmap v18: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 121 KiB/s rd, 3.0 MiB/s wr, 134 op/s 2024-03-29T04:54:39.871 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:39 smithi181 ceph-mon[29765]: osdmap e153: 8 total, 8 up, 8 in 2024-03-29T04:54:39.872 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:39 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:39.872 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:39 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:40.793 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:40 smithi187 ceph-mon[33595]: osdmap e154: 8 total, 8 up, 8 in 2024-03-29T04:54:40.793 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:40 smithi187 ceph-mon[33595]: pgmap v21: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 490 op/s 2024-03-29T04:54:40.793 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:40 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2436970188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:40.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:40 smithi181 ceph-mon[29765]: osdmap e154: 8 total, 8 up, 8 in 2024-03-29T04:54:40.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:40 smithi181 ceph-mon[29765]: pgmap v21: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 490 op/s 2024-03-29T04:54:40.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:40 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2436970188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:40.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:40 smithi181 ceph-mon[41183]: osdmap e154: 8 total, 8 up, 8 in 2024-03-29T04:54:40.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:40 smithi181 ceph-mon[41183]: pgmap v21: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 490 op/s 2024-03-29T04:54:40.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:40 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2436970188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:41.698 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 8388609~1000 2024-03-29T04:54:41.705 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 16777228~1000 2024-03-29T04:54:41.705 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 167772160 2024-03-29T04:54:41.713 INFO:tasks.workunit.client.0.smithi181.stdout:write: 2 83886079~8388609 2024-03-29T04:54:41.728 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap3 2024-03-29T04:54:41.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:41 smithi187 ceph-mon[33595]: osdmap e155: 8 total, 8 up, 8 in 2024-03-29T04:54:41.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:41 smithi181 ceph-mon[41183]: osdmap e155: 8 total, 8 up, 8 in 2024-03-29T04:54:41.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:41 smithi181 ceph-mon[29765]: osdmap e155: 8 total, 8 up, 8 in 2024-03-29T04:54:42.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: osdmap e156: 8 total, 8 up, 8 in 2024-03-29T04:54:42.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: pgmap v24: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 490 op/s 2024-03-29T04:54:42.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3689707867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:42.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:42.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:42.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:42.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:42.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:42.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:42.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:42.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:42.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:42.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:42.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: Adjusting osd_memory_target on smithi187 to 2480M 2024-03-29T04:54:42.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:42.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:54:42.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:54:42.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: Updating smithi181:/etc/ceph/ceph.conf 2024-03-29T04:54:42.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:54:42.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[41183]: osdmap e157: 8 total, 8 up, 8 in 2024-03-29T04:54:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: osdmap e156: 8 total, 8 up, 8 in 2024-03-29T04:54:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: pgmap v24: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 490 op/s 2024-03-29T04:54:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3689707867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:42.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: Adjusting osd_memory_target on smithi187 to 2480M 2024-03-29T04:54:42.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:42.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:54:42.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:54:42.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: Updating smithi181:/etc/ceph/ceph.conf 2024-03-29T04:54:42.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:54:42.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:42 smithi181 ceph-mon[29765]: osdmap e157: 8 total, 8 up, 8 in 2024-03-29T04:54:43.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: osdmap e156: 8 total, 8 up, 8 in 2024-03-29T04:54:43.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: pgmap v24: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 490 op/s 2024-03-29T04:54:43.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3689707867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:43.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:43.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:43.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:43.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:43.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:43.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:43.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:43.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:54:43.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: Adjusting osd_memory_target on smithi187 to 2480M 2024-03-29T04:54:43.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:54:43.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:54:43.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: Updating smithi181:/etc/ceph/ceph.conf 2024-03-29T04:54:43.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:54:43.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:42 smithi187 ceph-mon[33595]: osdmap e157: 8 total, 8 up, 8 in 2024-03-29T04:54:43.480 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:54:43.509 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 83886079~1 2024-03-29T04:54:43.516 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:54:43.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:54:43.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:54:43.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:43.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:54:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:54:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:54:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:54:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: osdmap e158: 8 total, 8 up, 8 in 2024-03-29T04:54:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:54:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:54:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:54:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:54:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:54:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:54:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: osdmap e158: 8 total, 8 up, 8 in 2024-03-29T04:54:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:43.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:43 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:44.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:54:44.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:54:44.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:44.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:54:44.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:54:44.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:54:44.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:54:44.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: osdmap e158: 8 total, 8 up, 8 in 2024-03-29T04:54:44.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:44.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:44.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:44.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:44.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:44.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:44.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:44.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:44.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:43 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:44.701 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:44 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:54:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:54:45.673 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:45 smithi181 ceph-mon[29765]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:54:45.673 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:45 smithi181 ceph-mon[29765]: Reconfiguring daemon alertmanager.a on smithi181 2024-03-29T04:54:45.673 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:45 smithi181 ceph-mon[29765]: pgmap v27: 137 pgs: 137 active+clean; 421 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s wr, 1 op/s 2024-03-29T04:54:45.673 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:45 smithi181 ceph-mon[29765]: osdmap e159: 8 total, 8 up, 8 in 2024-03-29T04:54:45.673 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:45 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/674283867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:45.674 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:45 smithi181 ceph-mon[41183]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:54:45.674 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:45 smithi181 ceph-mon[41183]: Reconfiguring daemon alertmanager.a on smithi181 2024-03-29T04:54:45.674 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:45 smithi181 ceph-mon[41183]: pgmap v27: 137 pgs: 137 active+clean; 421 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s wr, 1 op/s 2024-03-29T04:54:45.674 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:45 smithi181 ceph-mon[41183]: osdmap e159: 8 total, 8 up, 8 in 2024-03-29T04:54:45.674 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:45 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/674283867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:45.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:45 smithi187 ceph-mon[33595]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:54:45.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:45 smithi187 ceph-mon[33595]: Reconfiguring daemon alertmanager.a on smithi181 2024-03-29T04:54:45.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:45 smithi187 ceph-mon[33595]: pgmap v27: 137 pgs: 137 active+clean; 421 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s wr, 1 op/s 2024-03-29T04:54:45.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:45 smithi187 ceph-mon[33595]: osdmap e159: 8 total, 8 up, 8 in 2024-03-29T04:54:45.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:45 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/674283867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:46.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:46 smithi181 ceph-mon[29765]: osdmap e160: 8 total, 8 up, 8 in 2024-03-29T04:54:46.701 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:46 smithi181 ceph-mon[41183]: osdmap e160: 8 total, 8 up, 8 in 2024-03-29T04:54:46.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:46 smithi187 ceph-mon[33595]: osdmap e160: 8 total, 8 up, 8 in 2024-03-29T04:54:46.970 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 41943040 2024-03-29T04:54:46.999 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:54:47.150 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:54:47.201 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:46 smithi181 systemd[1]: Stopping Ceph alertmanager.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:54:47.520 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:47 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[78260]: ts=2024-03-29T04:54:47.224Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:54:47.520 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:47 smithi181 podman[91003]: 2024-03-29 04:54:47.265141074 +0000 UTC m=+0.180682723 container died d22dcca4e726cd0151679ffa1cdc5a9c41556c3cc5a36e66105f0d9c6c59d50e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:47.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:47 smithi187 ceph-mon[33595]: pgmap v30: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-03-29T04:54:47.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:47 smithi187 ceph-mon[33595]: osdmap e161: 8 total, 8 up, 8 in 2024-03-29T04:54:47.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:47 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2765772450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:47.886 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:47 smithi181 ceph-mon[29765]: pgmap v30: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-03-29T04:54:47.886 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:47 smithi181 ceph-mon[29765]: osdmap e161: 8 total, 8 up, 8 in 2024-03-29T04:54:47.887 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:47 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2765772450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:47.887 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:47 smithi181 podman[91003]: 2024-03-29 04:54:47.613949959 +0000 UTC m=+0.529491599 container cleanup d22dcca4e726cd0151679ffa1cdc5a9c41556c3cc5a36e66105f0d9c6c59d50e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:47.887 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:47 smithi181 bash[91003]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a 2024-03-29T04:54:47.887 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:47 smithi181 ceph-mon[41183]: pgmap v30: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-03-29T04:54:47.887 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:47 smithi181 ceph-mon[41183]: osdmap e161: 8 total, 8 up, 8 in 2024-03-29T04:54:47.887 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:47 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2765772450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:48.171 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:47 smithi181 podman[91022]: 2024-03-29 04:54:47.881850504 +0000 UTC m=+0.614923615 container remove d22dcca4e726cd0151679ffa1cdc5a9c41556c3cc5a36e66105f0d9c6c59d50e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:48.171 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:47 smithi181 podman[91022]: 2024-03-29 04:54:47.99113443 +0000 UTC m=+0.724207503 volume remove e8575d0e0bec37bd2f5e334c7deb5eae75090e23fe84ec8b06096f8f7699ae20 2024-03-29T04:54:48.451 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-03-29T04:54:48.451 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 systemd[1]: Stopped Ceph alertmanager.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:54:48.451 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@alertmanager.a.service: Consumed 1.181s CPU time. 2024-03-29T04:54:48.452 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 systemd[1]: Starting Ceph alertmanager.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:54:48.452 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 podman[91123]: 2024-03-29 04:54:48.291694272 +0000 UTC m=+0.017700313 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-03-29T04:54:48.452 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 podman[91123]: 2024-03-29 04:54:48.400353621 +0000 UTC m=+0.126359683 volume create f34a0c8bd3e80a46f5462c7d67e3353f87012b10067e4c3a01d363dcc0f0dab6 2024-03-29T04:54:48.850 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:48 smithi181 ceph-mon[41183]: osdmap e162: 8 total, 8 up, 8 in 2024-03-29T04:54:48.851 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 podman[91123]: 2024-03-29 04:54:48.484087598 +0000 UTC m=+0.210093615 container create 69e0ad67b57f879ac04b924eef88d502c74d8c54238bf566fc0aa4642ebf3fa2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:48.851 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:48 smithi181 ceph-mon[29765]: osdmap e162: 8 total, 8 up, 8 in 2024-03-29T04:54:48.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:48 smithi187 ceph-mon[33595]: osdmap e162: 8 total, 8 up, 8 in 2024-03-29T04:54:48.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:54:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:54:48] "GET /metrics HTTP/1.1" 200 38660 "" "Prometheus/2.43.0" 2024-03-29T04:54:49.201 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 podman[91123]: 2024-03-29 04:54:48.84572492 +0000 UTC m=+0.571730937 container init 69e0ad67b57f879ac04b924eef88d502c74d8c54238bf566fc0aa4642ebf3fa2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:49.202 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 podman[91123]: 2024-03-29 04:54:48.850769966 +0000 UTC m=+0.576775996 container start 69e0ad67b57f879ac04b924eef88d502c74d8c54238bf566fc0aa4642ebf3fa2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:49.202 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T04:54:48.862Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-03-29T04:54:49.202 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T04:54:48.862Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-03-29T04:54:49.202 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T04:54:48.863Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.181 port=9094 2024-03-29T04:54:49.202 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T04:54:48.864Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-03-29T04:54:49.202 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T04:54:48.887Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:54:49.202 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T04:54:48.888Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:54:49.202 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T04:54:48.890Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-03-29T04:54:49.202 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T04:54:48.890Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-03-29T04:54:49.202 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 bash[91123]: 69e0ad67b57f879ac04b924eef88d502c74d8c54238bf566fc0aa4642ebf3fa2 2024-03-29T04:54:49.202 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:48 smithi181 systemd[1]: Started Ceph alertmanager.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:54:49.833 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.SnapsShrinkAfterPrepare (12021 ms) 2024-03-29T04:54:49.833 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroBeforePrepare 2024-03-29T04:54:49.844 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:49 smithi181 systemd[1]: Stopping Ceph node-exporter.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:54:49.844 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:49 smithi181 ceph-mon[29765]: pgmap v33: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 140 KiB/s rd, 5.0 MiB/s wr, 200 op/s 2024-03-29T04:54:49.844 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:49 smithi181 ceph-mon[29765]: osdmap e163: 8 total, 8 up, 8 in 2024-03-29T04:54:49.844 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:49 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:49.844 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:49 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:49.845 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:49 smithi181 ceph-mon[41183]: pgmap v33: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 140 KiB/s rd, 5.0 MiB/s wr, 200 op/s 2024-03-29T04:54:49.845 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:49 smithi181 ceph-mon[41183]: osdmap e163: 8 total, 8 up, 8 in 2024-03-29T04:54:49.845 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:49 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:49.845 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:49 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:49.864 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:54:49.875 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:54:49.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:49 smithi187 ceph-mon[33595]: pgmap v33: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 140 KiB/s rd, 5.0 MiB/s wr, 200 op/s 2024-03-29T04:54:49.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:49 smithi187 ceph-mon[33595]: osdmap e163: 8 total, 8 up, 8 in 2024-03-29T04:54:49.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:49 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:49.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:49 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:49.935 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:54:50.147 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:49 smithi181 podman[91314]: 2024-03-29 04:54:49.838675337 +0000 UTC m=+0.153937851 container died 2f1dfb6435e1cf391299df6eeb07a9611f6c5083b617dea12954d3ab08b91e0f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:50.451 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:50 smithi181 podman[91314]: 2024-03-29 04:54:50.249849034 +0000 UTC m=+0.565111536 container cleanup 2f1dfb6435e1cf391299df6eeb07a9611f6c5083b617dea12954d3ab08b91e0f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:50.452 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:50 smithi181 bash[91314]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a 2024-03-29T04:54:50.870 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:50 smithi181 ceph-mon[29765]: Reconfiguring node-exporter.a (dependencies changed)... 2024-03-29T04:54:50.870 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:50 smithi181 ceph-mon[29765]: Reconfiguring daemon node-exporter.a on smithi181 2024-03-29T04:54:50.870 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:50 smithi181 ceph-mon[29765]: osdmap e164: 8 total, 8 up, 8 in 2024-03-29T04:54:50.870 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:50 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2141430439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:50.870 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:50 smithi181 ceph-mon[41183]: Reconfiguring node-exporter.a (dependencies changed)... 2024-03-29T04:54:50.870 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:50 smithi181 ceph-mon[41183]: Reconfiguring daemon node-exporter.a on smithi181 2024-03-29T04:54:50.870 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:50 smithi181 ceph-mon[41183]: osdmap e164: 8 total, 8 up, 8 in 2024-03-29T04:54:50.870 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:50 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2141430439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:50.871 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:50 smithi181 podman[91336]: 2024-03-29 04:54:50.467693701 +0000 UTC m=+0.633620093 container remove 2f1dfb6435e1cf391299df6eeb07a9611f6c5083b617dea12954d3ab08b91e0f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:50.871 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:50 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:54:50.871 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:50 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@node-exporter.a.service: Failed with result 'exit-code'. 2024-03-29T04:54:50.871 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:50 smithi181 systemd[1]: Stopped Ceph node-exporter.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:54:50.871 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:50 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@node-exporter.a.service: Consumed 2.210s CPU time. 2024-03-29T04:54:50.871 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:50 smithi181 systemd[1]: Starting Ceph node-exporter.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:54:50.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:50 smithi187 ceph-mon[33595]: Reconfiguring node-exporter.a (dependencies changed)... 2024-03-29T04:54:50.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:50 smithi187 ceph-mon[33595]: Reconfiguring daemon node-exporter.a on smithi181 2024-03-29T04:54:50.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:50 smithi187 ceph-mon[33595]: osdmap e164: 8 total, 8 up, 8 in 2024-03-29T04:54:50.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:50 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2141430439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:51.123 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:50 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T04:54:50.865Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000320586s 2024-03-29T04:54:51.123 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:50 smithi181 podman[91435]: 2024-03-29 04:54:50.793172564 +0000 UTC m=+0.017346935 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-03-29T04:54:51.123 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:50 smithi181 podman[91435]: 2024-03-29 04:54:50.893731741 +0000 UTC m=+0.117906089 container create 4029d029600b7db832598baa93643bf1952b1b4c8ba1dc65a966a1223b6924aa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:51.123 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 podman[91435]: 2024-03-29 04:54:51.112415738 +0000 UTC m=+0.336590083 container init 4029d029600b7db832598baa93643bf1952b1b4c8ba1dc65a966a1223b6924aa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:51.123 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 podman[91435]: 2024-03-29 04:54:51.118036712 +0000 UTC m=+0.342211056 container start 4029d029600b7db832598baa93643bf1952b1b4c8ba1dc65a966a1223b6924aa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:51.452 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.121Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-03-29T04:54:51.452 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.121Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-03-29T04:54:51.452 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.121Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-03-29T04:54:51.452 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.121Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-03-29T04:54:51.452 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-03-29T04:54:51.452 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-03-29T04:54:51.452 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-03-29T04:54:51.452 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=arp 2024-03-29T04:54:51.452 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=bcache 2024-03-29T04:54:51.452 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=bonding 2024-03-29T04:54:51.452 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=btrfs 2024-03-29T04:54:51.452 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=conntrack 2024-03-29T04:54:51.453 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=cpu 2024-03-29T04:54:51.453 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-03-29T04:54:51.453 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=diskstats 2024-03-29T04:54:51.453 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=dmi 2024-03-29T04:54:51.453 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=edac 2024-03-29T04:54:51.453 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=entropy 2024-03-29T04:54:51.453 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-03-29T04:54:51.454 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=filefd 2024-03-29T04:54:51.454 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=filesystem 2024-03-29T04:54:51.454 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=hwmon 2024-03-29T04:54:51.454 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=infiniband 2024-03-29T04:54:51.454 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=ipvs 2024-03-29T04:54:51.454 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=loadavg 2024-03-29T04:54:51.454 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=mdadm 2024-03-29T04:54:51.454 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=meminfo 2024-03-29T04:54:51.454 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=netclass 2024-03-29T04:54:51.454 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=netdev 2024-03-29T04:54:51.454 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=netstat 2024-03-29T04:54:51.454 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=nfs 2024-03-29T04:54:51.455 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=nfsd 2024-03-29T04:54:51.455 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=nvme 2024-03-29T04:54:51.455 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=os 2024-03-29T04:54:51.455 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-03-29T04:54:51.455 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=pressure 2024-03-29T04:54:51.455 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=rapl 2024-03-29T04:54:51.455 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=schedstat 2024-03-29T04:54:51.455 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=selinux 2024-03-29T04:54:51.455 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=sockstat 2024-03-29T04:54:51.455 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=softnet 2024-03-29T04:54:51.455 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=stat 2024-03-29T04:54:51.455 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=tapestats 2024-03-29T04:54:51.455 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=textfile 2024-03-29T04:54:51.455 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-03-29T04:54:51.456 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=time 2024-03-29T04:54:51.456 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-03-29T04:54:51.456 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=uname 2024-03-29T04:54:51.456 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=vmstat 2024-03-29T04:54:51.456 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=xfs 2024-03-29T04:54:51.456 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.122Z caller=node_exporter.go:117 level=info collector=zfs 2024-03-29T04:54:51.456 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.123Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-03-29T04:54:51.456 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a[91449]: ts=2024-03-29T04:54:51.123Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-03-29T04:54:51.456 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 bash[91435]: 4029d029600b7db832598baa93643bf1952b1b4c8ba1dc65a966a1223b6924aa 2024-03-29T04:54:51.456 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 04:54:51 smithi181 systemd[1]: Started Ceph node-exporter.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:54:51.558 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 8388609~8388609 2024-03-29T04:54:51.577 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 0~1000 2024-03-29T04:54:51.577 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 1010~1000 2024-03-29T04:54:51.577 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap2 2024-03-29T04:54:51.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:51 smithi187 ceph-mon[33595]: pgmap v36: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 430 op/s 2024-03-29T04:54:51.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:51 smithi187 ceph-mon[33595]: osdmap e165: 8 total, 8 up, 8 in 2024-03-29T04:54:51.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:51 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:51.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:51 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:51.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-mon[41183]: pgmap v36: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 430 op/s 2024-03-29T04:54:51.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-mon[41183]: osdmap e165: 8 total, 8 up, 8 in 2024-03-29T04:54:51.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:51.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:51.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-mon[29765]: pgmap v36: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 430 op/s 2024-03-29T04:54:51.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-mon[29765]: osdmap e165: 8 total, 8 up, 8 in 2024-03-29T04:54:51.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:51.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:51 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:52.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:52 smithi187 ceph-mon[33595]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:54:52.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:52 smithi187 ceph-mon[33595]: Reconfiguring daemon prometheus.a on smithi187 2024-03-29T04:54:52.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:52 smithi187 ceph-mon[33595]: osdmap e166: 8 total, 8 up, 8 in 2024-03-29T04:54:52.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:52 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2766132160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:52.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:52 smithi187 ceph-mon[33595]: osdmap e167: 8 total, 8 up, 8 in 2024-03-29T04:54:52.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:52 smithi181 ceph-mon[29765]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:54:52.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:52 smithi181 ceph-mon[29765]: Reconfiguring daemon prometheus.a on smithi187 2024-03-29T04:54:52.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:52 smithi181 ceph-mon[29765]: osdmap e166: 8 total, 8 up, 8 in 2024-03-29T04:54:52.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:52 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2766132160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:52.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:52 smithi181 ceph-mon[29765]: osdmap e167: 8 total, 8 up, 8 in 2024-03-29T04:54:52.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:52 smithi181 ceph-mon[41183]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:54:52.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:52 smithi181 ceph-mon[41183]: Reconfiguring daemon prometheus.a on smithi187 2024-03-29T04:54:52.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:52 smithi181 ceph-mon[41183]: osdmap e166: 8 total, 8 up, 8 in 2024-03-29T04:54:52.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:52 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2766132160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:52.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:52 smithi181 ceph-mon[41183]: osdmap e167: 8 total, 8 up, 8 in 2024-03-29T04:54:53.496 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 8388609~1000 2024-03-29T04:54:53.501 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 16777228~1000 2024-03-29T04:54:53.501 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 167772160 2024-03-29T04:54:53.512 INFO:tasks.workunit.client.0.smithi181.stdout:write: 2 83886079~8388609 2024-03-29T04:54:53.527 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap3 2024-03-29T04:54:53.813 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:53 smithi187 ceph-mon[33595]: pgmap v39: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 430 op/s 2024-03-29T04:54:53.813 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:53 smithi187 ceph-mon[33595]: osdmap e168: 8 total, 8 up, 8 in 2024-03-29T04:54:53.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:53 smithi181 ceph-mon[29765]: pgmap v39: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 430 op/s 2024-03-29T04:54:53.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:53 smithi181 ceph-mon[29765]: osdmap e168: 8 total, 8 up, 8 in 2024-03-29T04:54:53.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:53 smithi181 ceph-mon[41183]: pgmap v39: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 430 op/s 2024-03-29T04:54:53.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:53 smithi181 ceph-mon[41183]: osdmap e168: 8 total, 8 up, 8 in 2024-03-29T04:54:54.808 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:54 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3796459484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:54.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:54 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3796459484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:54.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:54 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3796459484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:54.952 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:54:54 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[30019]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:54:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:54:55.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:54 smithi187 systemd[1]: Stopping Ceph prometheus.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:54:55.564 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:54:55.218Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:54:55.564 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:54:55.218Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-29T04:54:55.564 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:54:55.218Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-29T04:54:55.564 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:54:55.218Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-29T04:54:55.564 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:54:55.218Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-29T04:54:55.564 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:54:55.218Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-29T04:54:55.564 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:54:55.218Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-29T04:54:55.564 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:54:55.218Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-29T04:54:55.565 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:54:55.218Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-29T04:54:55.565 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:54:55.218Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-29T04:54:55.565 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:54:55.218Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-29T04:54:55.565 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[67065]: ts=2024-03-29T04:54:55.218Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-29T04:54:55.565 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 podman[78963]: 2024-03-29 04:54:55.228768412 +0000 UTC m=+0.148322885 container died f7620d3fb68f42e2e270de1a462c2e3a2cf697ea53ddc39a690904bedd1e5546 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:55.573 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:54:55.593 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 83886079~1 2024-03-29T04:54:55.601 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 0 2024-03-29T04:54:55.616 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:54:55.863 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:55 smithi187 ceph-mon[33595]: pgmap v42: 137 pgs: 137 active+clean; 473 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.0 MiB/s wr, 39 op/s 2024-03-29T04:54:55.863 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:55 smithi187 ceph-mon[33595]: osdmap e169: 8 total, 8 up, 8 in 2024-03-29T04:54:55.864 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 podman[78963]: 2024-03-29 04:54:55.648578083 +0000 UTC m=+0.568132554 container cleanup f7620d3fb68f42e2e270de1a462c2e3a2cf697ea53ddc39a690904bedd1e5546 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:55.864 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 bash[78963]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a 2024-03-29T04:54:55.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:55 smithi181 ceph-mon[29765]: pgmap v42: 137 pgs: 137 active+clean; 473 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.0 MiB/s wr, 39 op/s 2024-03-29T04:54:55.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:55 smithi181 ceph-mon[29765]: osdmap e169: 8 total, 8 up, 8 in 2024-03-29T04:54:55.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:55 smithi181 ceph-mon[41183]: pgmap v42: 137 pgs: 137 active+clean; 473 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.0 MiB/s wr, 39 op/s 2024-03-29T04:54:55.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:55 smithi181 ceph-mon[41183]: osdmap e169: 8 total, 8 up, 8 in 2024-03-29T04:54:56.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 podman[78982]: 2024-03-29 04:54:55.858232439 +0000 UTC m=+0.628923122 container remove f7620d3fb68f42e2e270de1a462c2e3a2cf697ea53ddc39a690904bedd1e5546 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:56.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-03-29T04:54:56.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 systemd[1]: Stopped Ceph prometheus.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:54:56.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@prometheus.a.service: Consumed 1.706s CPU time. 2024-03-29T04:54:56.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:55 smithi187 systemd[1]: Starting Ceph prometheus.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:54:56.483 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 podman[79087]: 2024-03-29 04:54:56.094685977 +0000 UTC m=+0.018383218 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-03-29T04:54:56.483 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 podman[79087]: 2024-03-29 04:54:56.234353498 +0000 UTC m=+0.158050747 container create b1ce8fda3ac677a95a468e35927931fb0338fa64782c5994a01dc920daa39710 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:56.483 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 podman[79087]: 2024-03-29 04:54:56.478485398 +0000 UTC m=+0.402182639 container init b1ce8fda3ac677a95a468e35927931fb0338fa64782c5994a01dc920daa39710 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:56.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-mon[33595]: osdmap e170: 8 total, 8 up, 8 in 2024-03-29T04:54:56.882 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 podman[79087]: 2024-03-29 04:54:56.484219552 +0000 UTC m=+0.407916793 container start b1ce8fda3ac677a95a468e35927931fb0338fa64782c5994a01dc920daa39710 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:54:56.882 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.507Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-29T04:54:56.882 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.507Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-29T04:54:56.882 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.507Z caller=main.go:570 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi187 (none))" 2024-03-29T04:54:56.882 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.507Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-29T04:54:56.882 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.507Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-29T04:54:56.882 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.509Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-29T04:54:56.882 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.509Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-29T04:54:56.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.510Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-29T04:54:56.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.510Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-29T04:54:56.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.512Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-29T04:54:56.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.512Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.471µs 2024-03-29T04:54:56.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.512Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-29T04:54:56.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.513Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-03-29T04:54:56.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.527Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-03-29T04:54:56.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.528Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-03-29T04:54:56.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.528Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.327µs wal_replay_duration=15.217391ms wbl_replay_duration=120ns total_replay_duration=15.255098ms 2024-03-29T04:54:56.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.530Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-29T04:54:56.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.530Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-29T04:54:56.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.530Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-29T04:54:56.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.549Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.372249ms db_storage=890ns remote_storage=1.116µs web_handler=375ns query_engine=557ns scrape=361.491µs scrape_sd=72.881µs notify=10.128µs notify_sd=9.634µs rules=17.628749ms tracing=7.135µs 2024-03-29T04:54:56.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.549Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-29T04:54:56.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:54:56.549Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-29T04:54:56.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 bash[79087]: b1ce8fda3ac677a95a468e35927931fb0338fa64782c5994a01dc920daa39710 2024-03-29T04:54:56.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:54:56 smithi187 systemd[1]: Started Ceph prometheus.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:54:56.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:56 smithi181 ceph-mon[29765]: osdmap e170: 8 total, 8 up, 8 in 2024-03-29T04:54:56.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:56 smithi181 ceph-mon[41183]: osdmap e170: 8 total, 8 up, 8 in 2024-03-29T04:54:57.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:57 smithi187 ceph-mon[33595]: pgmap v45: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 MiB/s wr, 193 op/s 2024-03-29T04:54:57.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:57 smithi187 ceph-mon[33595]: osdmap e171: 8 total, 8 up, 8 in 2024-03-29T04:54:57.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:57 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:57.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:57 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:57.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:57 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3995861991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:57.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:57 smithi187 ceph-mon[33595]: osdmap e172: 8 total, 8 up, 8 in 2024-03-29T04:54:57.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:57 smithi181 ceph-mon[41183]: pgmap v45: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 MiB/s wr, 193 op/s 2024-03-29T04:54:57.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:57 smithi181 ceph-mon[41183]: osdmap e171: 8 total, 8 up, 8 in 2024-03-29T04:54:57.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:57 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:57.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:57 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:57.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:57 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3995861991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:57.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:57 smithi181 ceph-mon[41183]: osdmap e172: 8 total, 8 up, 8 in 2024-03-29T04:54:57.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:57 smithi181 ceph-mon[29765]: pgmap v45: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 MiB/s wr, 193 op/s 2024-03-29T04:54:57.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:57 smithi181 ceph-mon[29765]: osdmap e171: 8 total, 8 up, 8 in 2024-03-29T04:54:57.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:57 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:57.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:57 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:54:57.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:57 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3995861991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:57.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:57 smithi181 ceph-mon[29765]: osdmap e172: 8 total, 8 up, 8 in 2024-03-29T04:54:58.843 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:54:58.873 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:58 smithi181 ceph-mon[29765]: Reconfiguring grafana.a (dependencies changed)... 2024-03-29T04:54:58.873 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:58 smithi181 ceph-mon[29765]: Reconfiguring daemon grafana.a on smithi187 2024-03-29T04:54:58.873 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:58 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:58.873 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:58 smithi181 ceph-mon[29765]: osdmap e173: 8 total, 8 up, 8 in 2024-03-29T04:54:58.874 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:58 smithi181 ceph-mon[41183]: Reconfiguring grafana.a (dependencies changed)... 2024-03-29T04:54:58.874 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:58 smithi181 ceph-mon[41183]: Reconfiguring daemon grafana.a on smithi187 2024-03-29T04:54:58.874 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:58 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:58.874 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:58 smithi181 ceph-mon[41183]: osdmap e173: 8 total, 8 up, 8 in 2024-03-29T04:54:58.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:58 smithi187 ceph-mon[33595]: Reconfiguring grafana.a (dependencies changed)... 2024-03-29T04:54:58.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:58 smithi187 ceph-mon[33595]: Reconfiguring daemon grafana.a on smithi187 2024-03-29T04:54:58.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:58 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:58.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:58 smithi187 ceph-mon[33595]: osdmap e173: 8 total, 8 up, 8 in 2024-03-29T04:54:58.978 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:54:59.201 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 04:54:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T04:54:58.868Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003804707s 2024-03-29T04:54:59.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:59 smithi187 ceph-mon[33595]: pgmap v48: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 26 MiB/s rd, 11 MiB/s wr, 153 op/s 2024-03-29T04:54:59.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:54:59 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1607434862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:59.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:59 smithi181 ceph-mon[41183]: pgmap v48: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 26 MiB/s rd, 11 MiB/s wr, 153 op/s 2024-03-29T04:54:59.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:54:59 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1607434862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:54:59.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:59 smithi181 ceph-mon[29765]: pgmap v48: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 26 MiB/s rd, 11 MiB/s wr, 153 op/s 2024-03-29T04:54:59.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:54:59 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1607434862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:00.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:00 smithi187 ceph-mon[33595]: osdmap e174: 8 total, 8 up, 8 in 2024-03-29T04:55:00.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:00 smithi181 ceph-mon[41183]: osdmap e174: 8 total, 8 up, 8 in 2024-03-29T04:55:00.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:00 smithi181 ceph-mon[29765]: osdmap e174: 8 total, 8 up, 8 in 2024-03-29T04:55:01.830 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.SnapsShrinkToZeroBeforePrepare (11998 ms) 2024-03-29T04:55:01.831 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroAfterPrepare 2024-03-29T04:55:01.861 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:55:01.872 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:55:01.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:01 smithi187 ceph-mon[33595]: pgmap v51: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 15 KiB/s wr, 234 op/s 2024-03-29T04:55:01.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:01 smithi187 ceph-mon[33595]: osdmap e175: 8 total, 8 up, 8 in 2024-03-29T04:55:01.926 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:55:01.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:01 smithi181 ceph-mon[29765]: pgmap v51: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 15 KiB/s wr, 234 op/s 2024-03-29T04:55:01.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:01 smithi181 ceph-mon[29765]: osdmap e175: 8 total, 8 up, 8 in 2024-03-29T04:55:01.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:01 smithi181 ceph-mon[41183]: pgmap v51: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 15 KiB/s wr, 234 op/s 2024-03-29T04:55:01.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:01 smithi181 ceph-mon[41183]: osdmap e175: 8 total, 8 up, 8 in 2024-03-29T04:55:02.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:02 smithi187 ceph-mon[33595]: osdmap e176: 8 total, 8 up, 8 in 2024-03-29T04:55:02.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:02 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2462936721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:02.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:02 smithi181 ceph-mon[41183]: osdmap e176: 8 total, 8 up, 8 in 2024-03-29T04:55:02.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:02 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2462936721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:02.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:02 smithi181 ceph-mon[29765]: osdmap e176: 8 total, 8 up, 8 in 2024-03-29T04:55:02.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:02 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2462936721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:03.628 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 8388609~8388609 2024-03-29T04:55:03.640 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 0~1000 2024-03-29T04:55:03.641 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 1010~1000 2024-03-29T04:55:03.641 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap2 2024-03-29T04:55:03.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:03 smithi187 ceph-mon[33595]: pgmap v54: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 15 KiB/s wr, 234 op/s 2024-03-29T04:55:03.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:03 smithi187 ceph-mon[33595]: osdmap e177: 8 total, 8 up, 8 in 2024-03-29T04:55:03.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:03 smithi181 ceph-mon[29765]: pgmap v54: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 15 KiB/s wr, 234 op/s 2024-03-29T04:55:03.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:03 smithi181 ceph-mon[29765]: osdmap e177: 8 total, 8 up, 8 in 2024-03-29T04:55:03.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:03 smithi181 ceph-mon[41183]: pgmap v54: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 15 KiB/s wr, 234 op/s 2024-03-29T04:55:03.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:03 smithi181 ceph-mon[41183]: osdmap e177: 8 total, 8 up, 8 in 2024-03-29T04:55:04.870 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:04 smithi187 ceph-mon[33595]: osdmap e178: 8 total, 8 up, 8 in 2024-03-29T04:55:04.870 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:04 smithi187 ceph-mon[33595]: pgmap v57: 137 pgs: 137 active+clean; 513 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.0 MiB/s wr, 21 op/s 2024-03-29T04:55:04.870 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:04 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2983069193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:04.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:04 smithi181 ceph-mon[29765]: osdmap e178: 8 total, 8 up, 8 in 2024-03-29T04:55:04.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:04 smithi181 ceph-mon[29765]: pgmap v57: 137 pgs: 137 active+clean; 513 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.0 MiB/s wr, 21 op/s 2024-03-29T04:55:04.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:04 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2983069193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:04.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:04 smithi181 ceph-mon[41183]: osdmap e178: 8 total, 8 up, 8 in 2024-03-29T04:55:04.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:04 smithi181 ceph-mon[41183]: pgmap v57: 137 pgs: 137 active+clean; 513 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.0 MiB/s wr, 21 op/s 2024-03-29T04:55:04.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:04 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2983069193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:05.640 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 8388609~1000 2024-03-29T04:55:05.647 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 16777228~1000 2024-03-29T04:55:05.647 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 167772160 2024-03-29T04:55:05.656 INFO:tasks.workunit.client.0.smithi181.stdout:write: 2 83886079~8388609 2024-03-29T04:55:05.684 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap3 2024-03-29T04:55:05.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:05 smithi187 ceph-mon[33595]: osdmap e179: 8 total, 8 up, 8 in 2024-03-29T04:55:05.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:05 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:55:05] "GET /metrics HTTP/1.1" 200 38658 "" "Prometheus/2.43.0" 2024-03-29T04:55:05.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:05 smithi181 ceph-mon[29765]: osdmap e179: 8 total, 8 up, 8 in 2024-03-29T04:55:05.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:05 smithi181 ceph-mon[41183]: osdmap e179: 8 total, 8 up, 8 in 2024-03-29T04:55:06.239 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:05 smithi187 systemd[1]: Stopping Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:55:06.577 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:06 smithi187 podman[79494]: 2024-03-29 04:55:06.233951592 +0000 UTC m=+0.128232050 container died 8bb6c690e335fe0559dedc92ba8a3ee9a2e7d0064fd9beae8fe3dc0ce2dae66b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, name=ubi8, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., release=239.1651231664, io.openshift.expose-services=, io.openshift.tags=base rhel8, description=Ceph Grafana Container, distribution-scope=public, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , vcs-type=git, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-29T04:55:06.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:06 smithi187 ceph-mon[33595]: osdmap e180: 8 total, 8 up, 8 in 2024-03-29T04:55:06.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:06 smithi187 ceph-mon[33595]: pgmap v60: 137 pgs: 137 active+clean; 537 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 30 MiB/s rd, 14 MiB/s wr, 400 op/s 2024-03-29T04:55:06.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:06 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/928624520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:06.882 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:06 smithi187 podman[79494]: 2024-03-29 04:55:06.721170847 +0000 UTC m=+0.615451305 container cleanup 8bb6c690e335fe0559dedc92ba8a3ee9a2e7d0064fd9beae8fe3dc0ce2dae66b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, description=Ceph Grafana Container, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, distribution-scope=public, io.openshift.expose-services=, name=ubi8) 2024-03-29T04:55:06.882 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:06 smithi187 bash[79494]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a 2024-03-29T04:55:06.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:06 smithi181 ceph-mon[29765]: osdmap e180: 8 total, 8 up, 8 in 2024-03-29T04:55:06.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:06 smithi181 ceph-mon[29765]: pgmap v60: 137 pgs: 137 active+clean; 537 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 30 MiB/s rd, 14 MiB/s wr, 400 op/s 2024-03-29T04:55:06.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:06 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/928624520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:06 smithi181 ceph-mon[41183]: osdmap e180: 8 total, 8 up, 8 in 2024-03-29T04:55:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:06 smithi181 ceph-mon[41183]: pgmap v60: 137 pgs: 137 active+clean; 537 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 30 MiB/s rd, 14 MiB/s wr, 400 op/s 2024-03-29T04:55:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:06 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/928624520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:07.354 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:07 smithi187 podman[79513]: 2024-03-29 04:55:07.005802579 +0000 UTC m=+0.775054455 container remove 8bb6c690e335fe0559dedc92ba8a3ee9a2e7d0064fd9beae8fe3dc0ce2dae66b (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , release=239.1651231664, name=ubi8, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, version=8.5, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-03-29T04:55:07.354 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:07 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:55:07.354 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:07 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:55:07.354 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:07 smithi187 systemd[1]: Stopped Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:07.354 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:07 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Consumed 5.276s CPU time. 2024-03-29T04:55:07.354 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:07 smithi187 systemd[1]: Starting Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:55:07.491 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:55:07.512 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 83886079~1 2024-03-29T04:55:07.519 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:55:07.614 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:07 smithi187 podman[79617]: 2024-03-29 04:55:07.348375431 +0000 UTC m=+0.109478560 container create c49f27388c5f3e943ea59d0af6be9551354c007d8c64d27bf78ad522cecaad27 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, release=239.1651231664, name=ubi8, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, version=8.5, io.openshift.tags=base rhel8, vcs-type=git, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc.) 2024-03-29T04:55:07.614 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:07 smithi187 podman[79617]: 2024-03-29 04:55:07.255786384 +0000 UTC m=+0.016889485 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:55:07.615 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:07 smithi187 podman[79617]: 2024-03-29 04:55:07.609427744 +0000 UTC m=+0.370530831 container init c49f27388c5f3e943ea59d0af6be9551354c007d8c64d27bf78ad522cecaad27 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, distribution-scope=public, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.component=ubi8-container, name=ubi8, io.openshift.expose-services=, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-03-29T04:55:07.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:07 smithi187 ceph-mon[33595]: osdmap e181: 8 total, 8 up, 8 in 2024-03-29T04:55:07.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:07 smithi187 ceph-mon[33595]: osdmap e182: 8 total, 8 up, 8 in 2024-03-29T04:55:07.882 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:07 smithi187 podman[79617]: 2024-03-29 04:55:07.615273065 +0000 UTC m=+0.376376139 container start c49f27388c5f3e943ea59d0af6be9551354c007d8c64d27bf78ad522cecaad27 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, com.redhat.component=ubi8-container, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, io.buildah.version=1.32.0, maintainer=Paul Cuzner , name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., release=239.1651231664, architecture=x86_64, build-date=2022-04-29T11:48:11.055897) 2024-03-29T04:55:07.882 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:07 smithi187 bash[79617]: c49f27388c5f3e943ea59d0af6be9551354c007d8c64d27bf78ad522cecaad27 2024-03-29T04:55:07.882 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:07 smithi187 systemd[1]: Started Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:07.882 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:07 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[79632]: logger=settings t=2024-03-29T04:55:07.77499415Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:55:07.882 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:07 smithi187 podman[79668]: 2024-03-29 04:55:07.805520826 +0000 UTC m=+0.019156096 container died c49f27388c5f3e943ea59d0af6be9551354c007d8c64d27bf78ad522cecaad27 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, distribution-scope=public, io.buildah.version=1.32.0, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, version=8.5, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-03-29T04:55:07.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:07 smithi181 ceph-mon[41183]: osdmap e181: 8 total, 8 up, 8 in 2024-03-29T04:55:07.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:07 smithi181 ceph-mon[41183]: osdmap e182: 8 total, 8 up, 8 in 2024-03-29T04:55:07.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:07 smithi181 ceph-mon[29765]: osdmap e181: 8 total, 8 up, 8 in 2024-03-29T04:55:07.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:07 smithi181 ceph-mon[29765]: osdmap e182: 8 total, 8 up, 8 in 2024-03-29T04:55:08.466 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:08 smithi187 podman[79668]: 2024-03-29 04:55:08.362577393 +0000 UTC m=+0.576212688 container remove c49f27388c5f3e943ea59d0af6be9551354c007d8c64d27bf78ad522cecaad27 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, description=Ceph Grafana Container, vcs-type=git, version=8.5, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64) 2024-03-29T04:55:08.466 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:08 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:55:08.751 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:08 smithi187 systemd[1]: Stopping Ceph node-exporter.b for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:55:08.751 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:08 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:55:08.751 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:55:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:55:09.030 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:08 smithi187 podman[79834]: 2024-03-29 04:55:08.745949726 +0000 UTC m=+0.138903754 container died 9528215f6d2a52dbe07c06073d33d265e3882af0c35a23ea02b0ba4b3b5ced07 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:55:09.031 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:08 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:09.031 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:08 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:09.031 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:08 smithi187 ceph-mon[33595]: Reconfiguring node-exporter.b (dependencies changed)... 2024-03-29T04:55:09.031 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:08 smithi187 ceph-mon[33595]: Reconfiguring daemon node-exporter.b on smithi187 2024-03-29T04:55:09.031 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:08 smithi187 ceph-mon[33595]: pgmap v63: 137 pgs: 137 active+clean; 537 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 12 MiB/s wr, 378 op/s 2024-03-29T04:55:09.031 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:08 smithi187 ceph-mon[33595]: osdmap e183: 8 total, 8 up, 8 in 2024-03-29T04:55:09.111 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:08 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:09.111 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:08 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:09.111 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:08 smithi181 ceph-mon[41183]: Reconfiguring node-exporter.b (dependencies changed)... 2024-03-29T04:55:09.111 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:08 smithi181 ceph-mon[41183]: Reconfiguring daemon node-exporter.b on smithi187 2024-03-29T04:55:09.111 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:08 smithi181 ceph-mon[41183]: pgmap v63: 137 pgs: 137 active+clean; 537 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 12 MiB/s wr, 378 op/s 2024-03-29T04:55:09.111 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:08 smithi181 ceph-mon[41183]: osdmap e183: 8 total, 8 up, 8 in 2024-03-29T04:55:09.111 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:08 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:09.112 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:08 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:09.112 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:08 smithi181 ceph-mon[29765]: Reconfiguring node-exporter.b (dependencies changed)... 2024-03-29T04:55:09.112 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:08 smithi181 ceph-mon[29765]: Reconfiguring daemon node-exporter.b on smithi187 2024-03-29T04:55:09.112 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:08 smithi181 ceph-mon[29765]: pgmap v63: 137 pgs: 137 active+clean; 537 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 12 MiB/s wr, 378 op/s 2024-03-29T04:55:09.112 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:08 smithi181 ceph-mon[29765]: osdmap e183: 8 total, 8 up, 8 in 2024-03-29T04:55:09.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:09 smithi187 podman[79834]: 2024-03-29 04:55:09.207271385 +0000 UTC m=+0.600225424 container cleanup 9528215f6d2a52dbe07c06073d33d265e3882af0c35a23ea02b0ba4b3b5ced07 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:55:09.382 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:09 smithi187 bash[79834]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b 2024-03-29T04:55:09.772 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:09 smithi187 podman[79853]: 2024-03-29 04:55:09.46701964 +0000 UTC m=+0.724281498 container remove 9528215f6d2a52dbe07c06073d33d265e3882af0c35a23ea02b0ba4b3b5ced07 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:55:09.772 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:09 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:55:09.772 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:09 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@node-exporter.b.service: Failed with result 'exit-code'. 2024-03-29T04:55:09.773 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:09 smithi187 systemd[1]: Stopped Ceph node-exporter.b for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:09.773 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:09 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@node-exporter.b.service: Consumed 2.199s CPU time. 2024-03-29T04:55:09.773 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:09 smithi187 systemd[1]: Starting Ceph node-exporter.b for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:55:10.032 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:09 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3313257584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:10.033 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:09 smithi181 ceph-mon[29765]: osdmap e184: 8 total, 8 up, 8 in 2024-03-29T04:55:10.033 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:09 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3313257584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:10.033 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:09 smithi181 ceph-mon[41183]: osdmap e184: 8 total, 8 up, 8 in 2024-03-29T04:55:10.108 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:09 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3313257584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:10.108 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:09 smithi187 ceph-mon[33595]: osdmap e184: 8 total, 8 up, 8 in 2024-03-29T04:55:10.109 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:09 smithi187 podman[79953]: 2024-03-29 04:55:09.731197929 +0000 UTC m=+0.017461147 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-03-29T04:55:10.109 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:09 smithi187 podman[79953]: 2024-03-29 04:55:09.868312082 +0000 UTC m=+0.154575334 container create 3894f6e97d55065f63c4fd150cff365ad2e04d6bb2c387aefe5ffa4ca019b59e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:55:10.109 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 podman[79953]: 2024-03-29 04:55:10.103403228 +0000 UTC m=+0.389666439 container init 3894f6e97d55065f63c4fd150cff365ad2e04d6bb2c387aefe5ffa4ca019b59e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:55:10.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 podman[79953]: 2024-03-29 04:55:10.108819165 +0000 UTC m=+0.395082378 container start 3894f6e97d55065f63c4fd150cff365ad2e04d6bb2c387aefe5ffa4ca019b59e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:55:10.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.112Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-03-29T04:55:10.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.112Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-03-29T04:55:10.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-03-29T04:55:10.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-03-29T04:55:10.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-03-29T04:55:10.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-03-29T04:55:10.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-03-29T04:55:10.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=arp 2024-03-29T04:55:10.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=bcache 2024-03-29T04:55:10.383 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=bonding 2024-03-29T04:55:10.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=btrfs 2024-03-29T04:55:10.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=conntrack 2024-03-29T04:55:10.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=cpu 2024-03-29T04:55:10.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-03-29T04:55:10.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=diskstats 2024-03-29T04:55:10.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=dmi 2024-03-29T04:55:10.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=edac 2024-03-29T04:55:10.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=entropy 2024-03-29T04:55:10.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-03-29T04:55:10.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=filefd 2024-03-29T04:55:10.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=filesystem 2024-03-29T04:55:10.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=hwmon 2024-03-29T04:55:10.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=infiniband 2024-03-29T04:55:10.384 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=ipvs 2024-03-29T04:55:10.385 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=loadavg 2024-03-29T04:55:10.385 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=mdadm 2024-03-29T04:55:10.385 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=meminfo 2024-03-29T04:55:10.385 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=netclass 2024-03-29T04:55:10.385 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=netdev 2024-03-29T04:55:10.385 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=netstat 2024-03-29T04:55:10.385 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=nfs 2024-03-29T04:55:10.385 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=nfsd 2024-03-29T04:55:10.385 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=nvme 2024-03-29T04:55:10.385 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=os 2024-03-29T04:55:10.385 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-03-29T04:55:10.385 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=pressure 2024-03-29T04:55:10.385 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=rapl 2024-03-29T04:55:10.385 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=schedstat 2024-03-29T04:55:10.385 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=selinux 2024-03-29T04:55:10.386 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=sockstat 2024-03-29T04:55:10.386 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=softnet 2024-03-29T04:55:10.386 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=stat 2024-03-29T04:55:10.386 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=tapestats 2024-03-29T04:55:10.386 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=textfile 2024-03-29T04:55:10.386 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-03-29T04:55:10.386 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=time 2024-03-29T04:55:10.386 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-03-29T04:55:10.386 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=uname 2024-03-29T04:55:10.386 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=vmstat 2024-03-29T04:55:10.386 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=xfs 2024-03-29T04:55:10.386 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=node_exporter.go:117 level=info collector=zfs 2024-03-29T04:55:10.386 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-03-29T04:55:10.386 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b[79968]: ts=2024-03-29T04:55:10.113Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-03-29T04:55:10.387 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 bash[79953]: 3894f6e97d55065f63c4fd150cff365ad2e04d6bb2c387aefe5ffa4ca019b59e 2024-03-29T04:55:10.387 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 04:55:10 smithi187 systemd[1]: Started Ceph node-exporter.b for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:10.926 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 0 2024-03-29T04:55:10.999 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:55:11.120 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:55:11.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:11 smithi187 ceph-mon[33595]: pgmap v66: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-03-29T04:55:11.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:11 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:11.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:11 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:11.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:11 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:55:11.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:11 smithi187 ceph-mon[33595]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:55:11.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:11 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:55:11.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:11 smithi187 ceph-mon[33595]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:55:11.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:11 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:55:11.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:11 smithi187 ceph-mon[33595]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:55:11.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:11 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:11.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:11 smithi187 ceph-mon[33595]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:55:11.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:11 smithi187 ceph-mon[33595]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:55:11.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:11 smithi187 ceph-mon[33595]: osdmap e185: 8 total, 8 up, 8 in 2024-03-29T04:55:11.633 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[41183]: pgmap v66: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-03-29T04:55:11.633 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:11.633 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:11.633 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:55:11.634 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[41183]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:55:11.634 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:55:11.634 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[41183]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:55:11.634 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:55:11.634 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[41183]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:55:11.634 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:11.634 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[41183]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:55:11.634 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[41183]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:55:11.634 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[41183]: osdmap e185: 8 total, 8 up, 8 in 2024-03-29T04:55:11.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[29765]: pgmap v66: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-03-29T04:55:11.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:11.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:11.635 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:55:11.635 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[29765]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:55:11.635 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:55:11.635 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[29765]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:55:11.635 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:55:11.635 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[29765]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:55:11.635 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:11.635 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[29765]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:55:11.635 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[29765]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:55:11.635 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:11 smithi181 ceph-mon[29765]: osdmap e185: 8 total, 8 up, 8 in 2024-03-29T04:55:12.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:12 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3750720731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:12.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:12 smithi181 ceph-mon[41183]: osdmap e186: 8 total, 8 up, 8 in 2024-03-29T04:55:12.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:12 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:12.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:12 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:55:12.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:12 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:55:12.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:12 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:55:12.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:12 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:12.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:12 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3750720731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:12.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:12 smithi181 ceph-mon[29765]: osdmap e186: 8 total, 8 up, 8 in 2024-03-29T04:55:12.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:12 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:12.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:12 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:55:12.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:12 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:55:12.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:12 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:55:12.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:12 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:12.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:12 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3750720731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:12.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:12 smithi187 ceph-mon[33595]: osdmap e186: 8 total, 8 up, 8 in 2024-03-29T04:55:12.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:12 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:12.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:12 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:55:12.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:12 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:55:12.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:12 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:55:12.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:12 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:13.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:13 smithi187 ceph-mon[33595]: pgmap v69: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-03-29T04:55:13.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:13 smithi187 ceph-mon[33595]: Upgrade: Updating mgr.y 2024-03-29T04:55:13.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:13 smithi187 ceph-mon[33595]: Deploying daemon mgr.y on smithi181 2024-03-29T04:55:13.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:13 smithi187 ceph-mon[33595]: osdmap e187: 8 total, 8 up, 8 in 2024-03-29T04:55:13.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:13 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:13.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:13 smithi181 ceph-mon[29765]: pgmap v69: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-03-29T04:55:13.701 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:13 smithi181 ceph-mon[29765]: Upgrade: Updating mgr.y 2024-03-29T04:55:13.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:13 smithi181 ceph-mon[29765]: Deploying daemon mgr.y on smithi181 2024-03-29T04:55:13.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:13 smithi181 ceph-mon[29765]: osdmap e187: 8 total, 8 up, 8 in 2024-03-29T04:55:13.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:13 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:13.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:13 smithi181 ceph-mon[41183]: pgmap v69: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-03-29T04:55:13.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:13 smithi181 ceph-mon[41183]: Upgrade: Updating mgr.y 2024-03-29T04:55:13.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:13 smithi181 ceph-mon[41183]: Deploying daemon mgr.y on smithi181 2024-03-29T04:55:13.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:13 smithi181 ceph-mon[41183]: osdmap e187: 8 total, 8 up, 8 in 2024-03-29T04:55:13.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:13 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:14.032 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.SnapsShrinkToZeroAfterPrepare (12201 ms) 2024-03-29T04:55:14.032 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare 2024-03-29T04:55:14.065 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:55:14.076 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:55:14.137 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:55:14.365 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:14 smithi181 systemd[1]: Stopping Ceph mgr.y for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:55:14.365 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:14 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2868976471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:14.365 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:14 smithi181 ceph-mon[29765]: osdmap e188: 8 total, 8 up, 8 in 2024-03-29T04:55:14.365 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:14 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2868976471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:14.366 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:14 smithi181 ceph-mon[41183]: osdmap e188: 8 total, 8 up, 8 in 2024-03-29T04:55:14.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:14 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2868976471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:14.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:14 smithi187 ceph-mon[33595]: osdmap e188: 8 total, 8 up, 8 in 2024-03-29T04:55:14.632 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:14 smithi181 podman[92096]: 2024-03-29 04:55:14.359326399 +0000 UTC m=+0.167000610 container died 37fa586e0972b1f4c4f2cd944e171c82fcaea660ae0d6ad53edca245ed94830d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-03-29T04:55:14.951 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:14 smithi181 podman[92096]: 2024-03-29 04:55:14.717266426 +0000 UTC m=+0.524940633 container cleanup 37fa586e0972b1f4c4f2cd944e171c82fcaea660ae0d6ad53edca245ed94830d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17) 2024-03-29T04:55:14.951 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:14 smithi181 bash[92096]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y 2024-03-29T04:55:15.278 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:14 smithi181 podman[92115]: 2024-03-29 04:55:14.985729776 +0000 UTC m=+0.625444875 container remove 37fa586e0972b1f4c4f2cd944e171c82fcaea660ae0d6ad53edca245ed94830d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-03-29T04:55:15.279 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:14 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:55:15.279 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:15 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.y.service: Failed with result 'exit-code'. 2024-03-29T04:55:15.279 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:15 smithi181 systemd[1]: Stopped Ceph mgr.y for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:15.279 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:15 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.y.service: Consumed 56.163s CPU time. 2024-03-29T04:55:15.279 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:15 smithi181 ceph-mon[29765]: pgmap v72: 137 pgs: 137 active+clean; 569 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.0 MiB/s wr, 10 op/s 2024-03-29T04:55:15.279 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:15 smithi181 ceph-mon[29765]: osdmap e189: 8 total, 8 up, 8 in 2024-03-29T04:55:15.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:15 smithi187 ceph-mon[33595]: pgmap v72: 137 pgs: 137 active+clean; 569 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.0 MiB/s wr, 10 op/s 2024-03-29T04:55:15.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:15 smithi187 ceph-mon[33595]: osdmap e189: 8 total, 8 up, 8 in 2024-03-29T04:55:15.646 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:15 smithi181 systemd[1]: Starting Ceph mgr.y for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:55:15.647 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:15 smithi181 podman[92253]: 2024-03-29 04:55:15.421274466 +0000 UTC m=+0.017910613 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:55:15.647 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:15 smithi181 podman[92253]: 2024-03-29 04:55:15.520847635 +0000 UTC m=+0.117483754 container create b07d84f4244a64d51076ac149797acee0011f1bb9f7faf2f8d6e0ad95ccb2005 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:55:15.647 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:15 smithi181 ceph-mon[41183]: pgmap v72: 137 pgs: 137 active+clean; 569 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.0 MiB/s wr, 10 op/s 2024-03-29T04:55:15.647 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:15 smithi181 ceph-mon[41183]: osdmap e189: 8 total, 8 up, 8 in 2024-03-29T04:55:15.811 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 8388609~8388609 2024-03-29T04:55:15.826 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 0~1000 2024-03-29T04:55:15.826 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 1010~1000 2024-03-29T04:55:15.826 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap2 2024-03-29T04:55:15.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:15 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:55:15] "GET /metrics HTTP/1.1" 200 38655 "" "Prometheus/2.43.0" 2024-03-29T04:55:16.201 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:15 smithi181 podman[92253]: 2024-03-29 04:55:15.740532164 +0000 UTC m=+0.337168284 container init b07d84f4244a64d51076ac149797acee0011f1bb9f7faf2f8d6e0ad95ccb2005 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-03-29T04:55:16.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:15 smithi181 podman[92253]: 2024-03-29 04:55:15.745555614 +0000 UTC m=+0.342191750 container start b07d84f4244a64d51076ac149797acee0011f1bb9f7faf2f8d6e0ad95ccb2005 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2) 2024-03-29T04:55:16.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:15 smithi181 ceph-mgr[92292]: -- 172.21.15.181:0/3182201459 <== mon.2 v2:172.21.15.181:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f0837ea5a0 con 0x55f0837bf000 2024-03-29T04:55:16.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:15 smithi181 bash[92253]: b07d84f4244a64d51076ac149797acee0011f1bb9f7faf2f8d6e0ad95ccb2005 2024-03-29T04:55:16.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:15 smithi181 systemd[1]: Started Ceph mgr.y for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:16.702 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:16 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:16.343+0000 7f300b345200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:55:16.702 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:16 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:16.445+0000 7f300b345200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:55:17.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:16 smithi187 ceph-mon[33595]: pgmap v74: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 581 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 42 MiB/s rd, 6.0 MiB/s wr, 448 op/s 2024-03-29T04:55:17.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:16 smithi187 ceph-mon[33595]: osdmap e190: 8 total, 8 up, 8 in 2024-03-29T04:55:17.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:16 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:17.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:16 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:17.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:16 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:17.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:16 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2006068810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:17.133 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:16 smithi181 ceph-mon[41183]: pgmap v74: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 581 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 42 MiB/s rd, 6.0 MiB/s wr, 448 op/s 2024-03-29T04:55:17.134 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:16 smithi181 ceph-mon[41183]: osdmap e190: 8 total, 8 up, 8 in 2024-03-29T04:55:17.134 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:16 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:17.134 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:16 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:17.134 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:16 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:17.134 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:16 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2006068810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:17.134 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:16 smithi181 ceph-mon[29765]: pgmap v74: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 581 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 42 MiB/s rd, 6.0 MiB/s wr, 448 op/s 2024-03-29T04:55:17.134 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:16 smithi181 ceph-mon[29765]: osdmap e190: 8 total, 8 up, 8 in 2024-03-29T04:55:17.134 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:16 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:17.134 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:16 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:17.134 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:16 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:17.134 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:16 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2006068810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:17.385 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:17 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:17.127+0000 7f300b345200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:55:17.385 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:17 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:17.227+0000 7f300b345200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:55:17.493 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 8388609~1000 2024-03-29T04:55:17.498 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 16777228~1000 2024-03-29T04:55:17.499 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 167772160 2024-03-29T04:55:17.507 INFO:tasks.workunit.client.0.smithi181.stdout:write: 2 83886079~8388609 2024-03-29T04:55:17.522 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap3 2024-03-29T04:55:17.677 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:17 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:17.379+0000 7f300b345200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:55:17.678 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:17 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:17.478+0000 7f300b345200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:55:17.678 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:17 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:17.584+0000 7f300b345200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:55:17.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:17 smithi187 ceph-mon[33595]: osdmap e191: 8 total, 8 up, 8 in 2024-03-29T04:55:17.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:17 smithi187 ceph-mon[33595]: osdmap e192: 8 total, 8 up, 8 in 2024-03-29T04:55:17.951 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:17 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:17.865+0000 7f300b345200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:55:17.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:17 smithi181 ceph-mon[29765]: osdmap e191: 8 total, 8 up, 8 in 2024-03-29T04:55:17.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:17 smithi181 ceph-mon[29765]: osdmap e192: 8 total, 8 up, 8 in 2024-03-29T04:55:17.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:17 smithi181 ceph-mon[41183]: osdmap e191: 8 total, 8 up, 8 in 2024-03-29T04:55:17.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:17 smithi181 ceph-mon[41183]: osdmap e192: 8 total, 8 up, 8 in 2024-03-29T04:55:18.398 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:17 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:17.977+0000 7f300b345200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:55:18.399 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:18.065+0000 7f300b345200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:55:18.399 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:18.176+0000 7f300b345200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:55:18.665 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:18.392+0000 7f300b345200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:55:19.045 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:18 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 1. 2024-03-29T04:55:19.045 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:18 smithi187 systemd[1]: Stopped Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:19.045 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:18 smithi187 systemd[1]: Starting Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:55:19.045 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:55:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:55:19.382 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:19 smithi187 podman[80495]: 2024-03-29 04:55:18.941706618 +0000 UTC m=+0.017598442 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:55:19.382 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:19 smithi187 podman[80495]: 2024-03-29 04:55:19.075711254 +0000 UTC m=+0.151603093 container create 60e72d60e00663000e723f6c90a3a2ce2ec9844b1b55c805fb52f4f561775f81 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, architecture=x86_64, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vcs-type=git, distribution-scope=public, io.buildah.version=1.32.0, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-03-29T04:55:19.501 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:55:19.525 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 83886079~1 2024-03-29T04:55:19.533 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:55:19.702 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:19 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:19.413+0000 7f300b345200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:55:19.702 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:19 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:19.569+0000 7f300b345200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:55:19.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:19 smithi181 ceph-mon[41183]: pgmap v78: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 581 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 4.0 MiB/s wr, 439 op/s 2024-03-29T04:55:19.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:19 smithi181 ceph-mon[41183]: osdmap e193: 8 total, 8 up, 8 in 2024-03-29T04:55:19.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:19 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2044102977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:19.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:19 smithi181 ceph-mon[29765]: pgmap v78: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 581 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 4.0 MiB/s wr, 439 op/s 2024-03-29T04:55:19.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:19 smithi181 ceph-mon[29765]: osdmap e193: 8 total, 8 up, 8 in 2024-03-29T04:55:19.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:19 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2044102977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:19.861 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:19 smithi187 ceph-mon[33595]: pgmap v78: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 581 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 4.0 MiB/s wr, 439 op/s 2024-03-29T04:55:19.862 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:19 smithi187 ceph-mon[33595]: osdmap e193: 8 total, 8 up, 8 in 2024-03-29T04:55:19.862 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:19 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2044102977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:19.862 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:19 smithi187 podman[80495]: 2024-03-29 04:55:19.377873083 +0000 UTC m=+0.453764886 container init 60e72d60e00663000e723f6c90a3a2ce2ec9844b1b55c805fb52f4f561775f81 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, vcs-type=git, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, architecture=x86_64, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, maintainer=Paul Cuzner , io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-03-29T04:55:19.862 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:19 smithi187 podman[80495]: 2024-03-29 04:55:19.383240305 +0000 UTC m=+0.459132113 container start 60e72d60e00663000e723f6c90a3a2ce2ec9844b1b55c805fb52f4f561775f81 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, name=ubi8, architecture=x86_64, io.openshift.expose-services=, distribution-scope=public, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-03-29T04:55:19.862 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:19 smithi187 bash[80495]: 60e72d60e00663000e723f6c90a3a2ce2ec9844b1b55c805fb52f4f561775f81 2024-03-29T04:55:19.862 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:19 smithi187 systemd[1]: Started Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:19.862 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:19 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[80520]: logger=settings t=2024-03-29T04:55:19.548865708Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:55:19.862 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:19 smithi187 podman[80578]: 2024-03-29 04:55:19.578803425 +0000 UTC m=+0.022531153 container died 60e72d60e00663000e723f6c90a3a2ce2ec9844b1b55c805fb52f4f561775f81 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, description=Ceph Grafana Container, version=8.5, distribution-scope=public, io.openshift.expose-services=, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-03-29T04:55:20.201 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:19 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:19.927+0000 7f300b345200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:55:20.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:20.025+0000 7f300b345200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:55:20.500 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:20.239+0000 7f300b345200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:55:20.500 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:20.439+0000 7f300b345200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:55:20.589 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:20 smithi187 ceph-mon[33595]: osdmap e194: 8 total, 8 up, 8 in 2024-03-29T04:55:20.590 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:20 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:20.590 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:20 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:20.590 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:20 smithi187 podman[80578]: 2024-03-29 04:55:20.248589146 +0000 UTC m=+0.692316848 container remove 60e72d60e00663000e723f6c90a3a2ce2ec9844b1b55c805fb52f4f561775f81 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.openshift.expose-services=, vendor=Red Hat, Inc., name=ubi8, distribution-scope=public, maintainer=Paul Cuzner , version=8.5, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0) 2024-03-29T04:55:20.590 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:20 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:55:20.758 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:20 smithi181 ceph-mon[41183]: osdmap e194: 8 total, 8 up, 8 in 2024-03-29T04:55:20.759 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:20 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:20.759 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:20 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:20.759 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:20 smithi181 ceph-mon[29765]: osdmap e194: 8 total, 8 up, 8 in 2024-03-29T04:55:20.759 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:20 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:20.759 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:20 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:20.882 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:20 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:55:21.034 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:20.752+0000 7f300b345200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:55:21.034 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:21.027+0000 7f300b345200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:55:21.684 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:21.285+0000 7f300b345200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:55:21.685 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:21 smithi181 ceph-mon[41183]: pgmap v81: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 194 op/s 2024-03-29T04:55:21.685 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:21 smithi181 ceph-mon[41183]: osdmap e195: 8 total, 8 up, 8 in 2024-03-29T04:55:21.685 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:21 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/4010890504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:21.685 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:21 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:21.685 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:21 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:21.685 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:21 smithi181 ceph-mon[29765]: pgmap v81: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 194 op/s 2024-03-29T04:55:21.685 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:21 smithi181 ceph-mon[29765]: osdmap e195: 8 total, 8 up, 8 in 2024-03-29T04:55:21.685 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:21 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/4010890504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:21.685 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:21 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:21.685 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:21 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:21.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:21 smithi187 ceph-mon[33595]: pgmap v81: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 194 op/s 2024-03-29T04:55:21.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:21 smithi187 ceph-mon[33595]: osdmap e195: 8 total, 8 up, 8 in 2024-03-29T04:55:21.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:21 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/4010890504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:21.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:21 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:21.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:21 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:22.404 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:22 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:22.159+0000 7f300b345200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:55:22.701 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:22 smithi181 ceph-mon[41183]: osdmap e196: 8 total, 8 up, 8 in 2024-03-29T04:55:22.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:22 smithi181 ceph-mon[41183]: osdmap e197: 8 total, 8 up, 8 in 2024-03-29T04:55:22.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:22 smithi181 ceph-mon[29765]: osdmap e196: 8 total, 8 up, 8 in 2024-03-29T04:55:22.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:22 smithi181 ceph-mon[29765]: osdmap e197: 8 total, 8 up, 8 in 2024-03-29T04:55:22.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:22 smithi187 ceph-mon[33595]: osdmap e196: 8 total, 8 up, 8 in 2024-03-29T04:55:22.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:22 smithi187 ceph-mon[33595]: osdmap e197: 8 total, 8 up, 8 in 2024-03-29T04:55:22.993 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 167772160 2024-03-29T04:55:23.045 INFO:tasks.workunit.client.0.smithi181.stdout:write: * 83886080~1000 2024-03-29T04:55:23.065 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:22 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:22.800+0000 7f300b345200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:55:23.065 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:22 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:22.935+0000 7f300b345200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:55:23.082 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:55:23.236 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:55:23.326 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:23.215+0000 7f300b345200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:55:23.326 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:55:23] ENGINE Bus STARTING 2024-03-29T04:55:23.702 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: CherryPy Checker: 2024-03-29T04:55:23.702 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: The Application mounted at '' has an empty config. 2024-03-29T04:55:23.702 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:23.702 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:55:23] ENGINE Serving on http://:::9283 2024-03-29T04:55:23.702 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:55:23] ENGINE Bus STARTED 2024-03-29T04:55:23.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-mon[29765]: pgmap v84: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 194 op/s 2024-03-29T04:55:23.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-mon[29765]: Standby manager daemon y restarted 2024-03-29T04:55:23.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-mon[29765]: Standby manager daemon y started 2024-03-29T04:55:23.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:55:23.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:55:23.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:55:23.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:55:23.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/124564022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:23.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-mon[41183]: pgmap v84: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 194 op/s 2024-03-29T04:55:23.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-mon[41183]: Standby manager daemon y restarted 2024-03-29T04:55:23.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-mon[41183]: Standby manager daemon y started 2024-03-29T04:55:23.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:55:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:55:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:55:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:55:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:23 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/124564022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:23.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:23 smithi187 ceph-mon[33595]: pgmap v84: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 194 op/s 2024-03-29T04:55:23.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:23 smithi187 ceph-mon[33595]: Standby manager daemon y restarted 2024-03-29T04:55:23.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:23 smithi187 ceph-mon[33595]: Standby manager daemon y started 2024-03-29T04:55:23.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:23 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:55:23.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:23 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:55:23.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:23 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:55:23.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:23 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:55:23.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:23 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/124564022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:24.773 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:24 smithi181 ceph-mon[29765]: mgrmap e26: x(active, since 55s), standbys: y 2024-03-29T04:55:24.773 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:24 smithi181 ceph-mon[29765]: osdmap e198: 8 total, 8 up, 8 in 2024-03-29T04:55:24.774 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:24 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:24.774 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:24 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:24.774 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:24 smithi181 ceph-mon[41183]: mgrmap e26: x(active, since 55s), standbys: y 2024-03-29T04:55:24.774 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:24 smithi181 ceph-mon[41183]: osdmap e198: 8 total, 8 up, 8 in 2024-03-29T04:55:24.774 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:24 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:24.774 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:24 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:24.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:24 smithi187 ceph-mon[33595]: mgrmap e26: x(active, since 55s), standbys: y 2024-03-29T04:55:24.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:24 smithi187 ceph-mon[33595]: osdmap e198: 8 total, 8 up, 8 in 2024-03-29T04:55:24.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:24 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:24.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:24 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:25.029 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:55:25] ENGINE Bus STOPPING 2024-03-29T04:55:25.263 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:24.997+0000 7f256b3fc700 -1 mgr handle_mgr_map I was active but no longer am 2024-03-29T04:55:25.263 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: ignoring --setuser ceph since I am not root 2024-03-29T04:55:25.263 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: ignoring --setgroup ceph since I am not root 2024-03-29T04:55:25.263 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mgr[70169]: -- 172.21.15.187:0/2088257750 <== mon.1 v2:172.21.15.187:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5627cfa1e5a0 con 0x5627cf9f3000 2024-03-29T04:55:25.335 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare (11303 ms) 2024-03-29T04:55:25.335 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare2 2024-03-29T04:55:25.366 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:55:25.377 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 41943040~10 2024-03-29T04:55:25.384 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:55:25.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:55:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:55:25.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:55:25] ENGINE Bus STOPPED 2024-03-29T04:55:25.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:55:25] ENGINE Bus STARTING 2024-03-29T04:55:25.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:25.174+0000 7f2f6b79d700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:55:25.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:25.174+0000 7f2f6b79d700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:55:25.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:25.174+0000 7f2f6b79d700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:55:25.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:25.174+0000 7f2f6b79d700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:55:25.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:25.174+0000 7f2f6b79d700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:55:25.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:25.178+0000 7f2f68f98700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:55:25.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:25.178+0000 7f2f68f98700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:55:25.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:25.178+0000 7f2f68f98700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:55:25.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:25.178+0000 7f2f68f98700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:55:25.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:55:25.178+0000 7f2f68f98700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:55:25.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:55:25] ENGINE Serving on http://:::9283 2024-03-29T04:55:25.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:55:25] ENGINE Bus STARTED 2024-03-29T04:55:25.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:25.598+0000 7f80d218f200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:55:25.883 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:25.695+0000 7f80d218f200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:55:25.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: pgmap v87: 137 pgs: 137 active+clean; 625 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 12 MiB/s rd, 3.0 MiB/s wr, 246 op/s 2024-03-29T04:55:25.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: osdmap e199: 8 total, 8 up, 8 in 2024-03-29T04:55:25.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:25.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:25.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:25.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:55:25.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:25.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:25.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:55:25.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:55:25.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: osdmap e200: 8 total, 8 up, 8 in 2024-03-29T04:55:25.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.24829 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-03-29T04:55:25.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: mgrmap e27: y(active, starting, since 0.00888385s) 2024-03-29T04:55:25.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:55:25.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:55:25.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:55:25.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi187.vcsnqx"}]: dispatch 2024-03-29T04:55:25.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.pflrwm"}]: dispatch 2024-03-29T04:55:25.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:55:25.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:55:25.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:55:25.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:55:25.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:55:25.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:55:25.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:55:25.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:55:25.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:55:25.888 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:55:25.888 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:55:25.888 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:55:25.888 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: Manager daemon y is now available 2024-03-29T04:55:25.888 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:25.888 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:25.888 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:55:25.889 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:25 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:55:25.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: pgmap v87: 137 pgs: 137 active+clean; 625 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 12 MiB/s rd, 3.0 MiB/s wr, 246 op/s 2024-03-29T04:55:25.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: osdmap e199: 8 total, 8 up, 8 in 2024-03-29T04:55:25.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:25.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:25.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:25.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:55:25.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:25.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:25.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:55:25.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:55:25.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: osdmap e200: 8 total, 8 up, 8 in 2024-03-29T04:55:25.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.24829 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-03-29T04:55:25.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: mgrmap e27: y(active, starting, since 0.00888385s) 2024-03-29T04:55:25.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:55:25.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:55:25.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:55:25.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi187.vcsnqx"}]: dispatch 2024-03-29T04:55:25.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.pflrwm"}]: dispatch 2024-03-29T04:55:25.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:55:25.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:55:25.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:55:25.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:55:25.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:55:25.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:55:25.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:55:25.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:55:25.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:55:25.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:55:25.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:55:25.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: pgmap v87: 137 pgs: 137 active+clean; 625 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 12 MiB/s rd, 3.0 MiB/s wr, 246 op/s 2024-03-29T04:55:25.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: osdmap e199: 8 total, 8 up, 8 in 2024-03-29T04:55:25.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:25.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:25.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:25.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:55:25.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' 2024-03-29T04:55:25.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:25.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.24829 172.21.15.187:0/4084356062' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:55:25.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:55:25.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: osdmap e200: 8 total, 8 up, 8 in 2024-03-29T04:55:25.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.24829 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-03-29T04:55:25.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: mgrmap e27: y(active, starting, since 0.00888385s) 2024-03-29T04:55:25.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:55:25.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:55:25.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:55:25.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi187.vcsnqx"}]: dispatch 2024-03-29T04:55:25.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.pflrwm"}]: dispatch 2024-03-29T04:55:25.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:55:25.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:55:25.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:55:25.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:55:25.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:55:25.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:55:25.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:55:25.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:55:25.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:55:25.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:55:25.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:55:25.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:55:25.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: Manager daemon y is now available 2024-03-29T04:55:25.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:25.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:25.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:55:25.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:55:25.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:55:25.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: Manager daemon y is now available 2024-03-29T04:55:25.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:25.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:25.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:55:25.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:25 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:55:26.632 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:26 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:26.395+0000 7f80d218f200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:55:26.632 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:26 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:26.490+0000 7f80d218f200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:55:26.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:26 smithi187 ceph-mon[33595]: [29/Mar/2024:04:55:25] ENGINE Bus STARTING 2024-03-29T04:55:26.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:26 smithi187 ceph-mon[33595]: [29/Mar/2024:04:55:25] ENGINE Serving on http://172.21.15.181:8765 2024-03-29T04:55:26.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:26 smithi187 ceph-mon[33595]: [29/Mar/2024:04:55:25] ENGINE Serving on https://172.21.15.181:7150 2024-03-29T04:55:26.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:26 smithi187 ceph-mon[33595]: [29/Mar/2024:04:55:25] ENGINE Bus STARTED 2024-03-29T04:55:26.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:26 smithi187 ceph-mon[33595]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:55:26.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:26 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/239883002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:26.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:26 smithi187 ceph-mon[33595]: mgrmap e28: y(active, since 1.00901s) 2024-03-29T04:55:26.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:26 smithi187 ceph-mon[33595]: osdmap e201: 8 total, 8 up, 8 in 2024-03-29T04:55:26.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:26 smithi181 ceph-mon[41183]: [29/Mar/2024:04:55:25] ENGINE Bus STARTING 2024-03-29T04:55:26.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:26 smithi181 ceph-mon[41183]: [29/Mar/2024:04:55:25] ENGINE Serving on http://172.21.15.181:8765 2024-03-29T04:55:26.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:26 smithi181 ceph-mon[41183]: [29/Mar/2024:04:55:25] ENGINE Serving on https://172.21.15.181:7150 2024-03-29T04:55:26.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:26 smithi181 ceph-mon[41183]: [29/Mar/2024:04:55:25] ENGINE Bus STARTED 2024-03-29T04:55:26.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:26 smithi181 ceph-mon[41183]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:55:26.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:26 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/239883002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:26.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:26 smithi181 ceph-mon[41183]: mgrmap e28: y(active, since 1.00901s) 2024-03-29T04:55:26.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:26 smithi181 ceph-mon[41183]: osdmap e201: 8 total, 8 up, 8 in 2024-03-29T04:55:26.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:26 smithi181 ceph-mon[29765]: [29/Mar/2024:04:55:25] ENGINE Bus STARTING 2024-03-29T04:55:26.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:26 smithi181 ceph-mon[29765]: [29/Mar/2024:04:55:25] ENGINE Serving on http://172.21.15.181:8765 2024-03-29T04:55:26.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:26 smithi181 ceph-mon[29765]: [29/Mar/2024:04:55:25] ENGINE Serving on https://172.21.15.181:7150 2024-03-29T04:55:26.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:26 smithi181 ceph-mon[29765]: [29/Mar/2024:04:55:25] ENGINE Bus STARTED 2024-03-29T04:55:26.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:26 smithi181 ceph-mon[29765]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:55:26.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:26 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/239883002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:26.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:26 smithi181 ceph-mon[29765]: mgrmap e28: y(active, since 1.00901s) 2024-03-29T04:55:26.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:26 smithi181 ceph-mon[29765]: osdmap e201: 8 total, 8 up, 8 in 2024-03-29T04:55:27.035 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 41943040 2024-03-29T04:55:27.052 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:55:27.125 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:26 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:26.635+0000 7f80d218f200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:55:27.125 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:26 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:26.732+0000 7f80d218f200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:55:27.125 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:26 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:26.830+0000 7f80d218f200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:55:27.125 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:27.118+0000 7f80d218f200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:55:27.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:27.237+0000 7f80d218f200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:55:27.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:27.326+0000 7f80d218f200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:55:27.644 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:55:27.655 INFO:tasks.workunit.client.0.smithi181.stdout:write: Y 41943040~5 2024-03-29T04:55:27.773 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:55:27.846 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:55:27.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:27 smithi187 ceph-mon[33595]: pgmap v4: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:27.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:27 smithi187 ceph-mon[33595]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:55:27.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:27 smithi187 ceph-mon[33595]: osdmap e202: 8 total, 8 up, 8 in 2024-03-29T04:55:27.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:27 smithi187 ceph-mon[33595]: osdmap e203: 8 total, 8 up, 8 in 2024-03-29T04:55:27.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:27.425+0000 7f80d218f200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:55:27.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:27.623+0000 7f80d218f200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:55:27.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:27 smithi181 ceph-mon[41183]: pgmap v4: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:27.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:27 smithi181 ceph-mon[41183]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:55:27.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:27 smithi181 ceph-mon[41183]: osdmap e202: 8 total, 8 up, 8 in 2024-03-29T04:55:27.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:27 smithi181 ceph-mon[41183]: osdmap e203: 8 total, 8 up, 8 in 2024-03-29T04:55:27.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:27 smithi181 ceph-mon[29765]: pgmap v4: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:27.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:27 smithi181 ceph-mon[29765]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:55:27.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:27 smithi181 ceph-mon[29765]: osdmap e202: 8 total, 8 up, 8 in 2024-03-29T04:55:27.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:27 smithi181 ceph-mon[29765]: osdmap e203: 8 total, 8 up, 8 in 2024-03-29T04:55:28.629 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare2 (3294 ms) 2024-03-29T04:55:28.629 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.SnapsSnapAfterPrepare 2024-03-29T04:55:28.658 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:55:28.669 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:55:28.730 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:55:28.856 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:28 smithi187 ceph-mon[33595]: pgmap v5: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:28.856 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:28 smithi187 ceph-mon[33595]: mgrmap e29: y(active, since 3s) 2024-03-29T04:55:28.856 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:28 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2701958811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:28.856 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:28.694+0000 7f80d218f200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:55:28.856 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:28.850+0000 7f80d218f200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:55:28.881 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:28 smithi181 ceph-mon[29765]: pgmap v5: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:28.882 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:28 smithi181 ceph-mon[29765]: mgrmap e29: y(active, since 3s) 2024-03-29T04:55:28.882 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:28 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2701958811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:28.882 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:28 smithi181 ceph-mon[41183]: pgmap v5: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:28.882 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:28 smithi181 ceph-mon[41183]: mgrmap e29: y(active, since 3s) 2024-03-29T04:55:28.882 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:28 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2701958811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:29.521 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:29 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:29.219+0000 7f80d218f200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:55:29.521 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:29 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:29.313+0000 7f80d218f200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:55:29.521 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:29 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:29.515+0000 7f80d218f200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:55:29.831 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:29 smithi187 ceph-mon[33595]: osdmap e204: 8 total, 8 up, 8 in 2024-03-29T04:55:29.831 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:29 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:29.832 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:29 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:29.832 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:29 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:29.743+0000 7f80d218f200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:55:29.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:29 smithi181 ceph-mon[29765]: osdmap e204: 8 total, 8 up, 8 in 2024-03-29T04:55:29.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:29 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:29.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:29 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:29.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:29 smithi181 ceph-mon[41183]: osdmap e204: 8 total, 8 up, 8 in 2024-03-29T04:55:29.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:29 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:29.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:29 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:30.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:30 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:30.056+0000 7f80d218f200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:55:30.565 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 8388609~8388609 2024-03-29T04:55:30.578 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 0~1000 2024-03-29T04:55:30.579 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 1010~1000 2024-03-29T04:55:30.579 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap2 2024-03-29T04:55:30.616 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:30 smithi187 ceph-mon[33595]: pgmap v9: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:30.617 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:30 smithi187 ceph-mon[33595]: osdmap e205: 8 total, 8 up, 8 in 2024-03-29T04:55:30.617 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:30 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:30.617 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:30 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:30.617 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:30 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:30.617 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:30 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:30.617 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:30 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:30.336+0000 7f80d218f200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:55:30.617 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:30 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:30.610+0000 7f80d218f200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:55:30.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:30 smithi181 ceph-mon[29765]: pgmap v9: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:30.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:30 smithi181 ceph-mon[29765]: osdmap e205: 8 total, 8 up, 8 in 2024-03-29T04:55:30.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:30 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:30.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:30 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:30.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:30 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:30.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:30 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:30 smithi181 ceph-mon[41183]: pgmap v9: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:30 smithi181 ceph-mon[41183]: osdmap e205: 8 total, 8 up, 8 in 2024-03-29T04:55:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:30 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:30 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:30 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:30 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:31.223 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:30 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 2. 2024-03-29T04:55:31.223 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:30 smithi187 systemd[1]: Stopped Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:31.224 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:30 smithi187 systemd[1]: Starting Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:55:31.224 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:31 smithi187 podman[81611]: 2024-03-29 04:55:31.016108676 +0000 UTC m=+0.017609729 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:55:31.493 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:31.466+0000 7f80d218f200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:55:31.494 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:31 smithi187 podman[81611]: 2024-03-29 04:55:31.21807633 +0000 UTC m=+0.219577367 container create ce01ca2e6fd590dfdc6707c9072b22f7d9091ad5655277068fa60a645e34a4f6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, distribution-scope=public, io.buildah.version=1.32.0, io.openshift.expose-services=, com.redhat.component=ubi8-container, name=ubi8, vcs-type=git, io.openshift.tags=base rhel8) 2024-03-29T04:55:31.494 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:31 smithi187 podman[81611]: 2024-03-29 04:55:31.487619706 +0000 UTC m=+0.489120770 container init ce01ca2e6fd590dfdc6707c9072b22f7d9091ad5655277068fa60a645e34a4f6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vendor=Red Hat, Inc., name=ubi8, vcs-type=git, distribution-scope=public, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5) 2024-03-29T04:55:31.701 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:31 smithi181 ceph-mon[41183]: osdmap e206: 8 total, 8 up, 8 in 2024-03-29T04:55:31.701 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:31 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/242390917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:31.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:31 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:31.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:31 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:31.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:31 smithi181 ceph-mon[29765]: osdmap e206: 8 total, 8 up, 8 in 2024-03-29T04:55:31.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:31 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/242390917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:31.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:31 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:31.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:31 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:31.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:31 smithi187 ceph-mon[33595]: osdmap e206: 8 total, 8 up, 8 in 2024-03-29T04:55:31.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:31 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/242390917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:31.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:31 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:31.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:31 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:31.882 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:31 smithi187 podman[81611]: 2024-03-29 04:55:31.494969998 +0000 UTC m=+0.496471035 container start ce01ca2e6fd590dfdc6707c9072b22f7d9091ad5655277068fa60a645e34a4f6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, io.buildah.version=1.32.0, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, com.redhat.component=ubi8-container, distribution-scope=public, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, name=ubi8, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container) 2024-03-29T04:55:31.882 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:31 smithi187 bash[81611]: ce01ca2e6fd590dfdc6707c9072b22f7d9091ad5655277068fa60a645e34a4f6 2024-03-29T04:55:31.883 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:31 smithi187 systemd[1]: Started Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:31.883 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[81627]: logger=settings t=2024-03-29T04:55:31.66977295Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:55:31.883 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:31 smithi187 podman[81664]: 2024-03-29 04:55:31.696438984 +0000 UTC m=+0.019593547 container died ce01ca2e6fd590dfdc6707c9072b22f7d9091ad5655277068fa60a645e34a4f6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, build-date=2022-04-29T11:48:11.055897, version=8.5, vcs-type=git, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, distribution-scope=public, io.openshift.expose-services=, release=239.1651231664, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, architecture=x86_64, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner ) 2024-03-29T04:55:32.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:32 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:32.079+0000 7f80d218f200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:55:32.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:32 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:32.179+0000 7f80d218f200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:55:32.583 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 8388609~1000 2024-03-29T04:55:32.587 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 16777228~1000 2024-03-29T04:55:32.587 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 167772160 2024-03-29T04:55:32.595 INFO:tasks.workunit.client.0.smithi181.stdout:write: 2 83886079~8388609 2024-03-29T04:55:32.609 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap3 2024-03-29T04:55:32.746 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:32 smithi187 podman[81664]: 2024-03-29 04:55:32.509077918 +0000 UTC m=+0.832232530 container remove ce01ca2e6fd590dfdc6707c9072b22f7d9091ad5655277068fa60a645e34a4f6 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, name=ubi8, release=239.1651231664, description=Ceph Grafana Container, vcs-type=git, io.openshift.expose-services=, maintainer=Paul Cuzner , architecture=x86_64, vendor=Red Hat, Inc.) 2024-03-29T04:55:32.746 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:32 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:55:32.747 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:32 smithi187 ceph-mon[33595]: pgmap v12: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:32.747 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:32 smithi187 ceph-mon[33595]: osdmap e207: 8 total, 8 up, 8 in 2024-03-29T04:55:32.747 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:32 smithi187 ceph-mon[33595]: Standby manager daemon x started 2024-03-29T04:55:32.747 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:32 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/310053580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:55:32.747 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:32 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/310053580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:55:32.747 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:32 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/310053580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:55:32.747 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:32 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/310053580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:55:32.747 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:32 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:32.407+0000 7f80d218f200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:55:32.747 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:32 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: [29/Mar/2024:04:55:32] ENGINE Bus STARTING 2024-03-29T04:55:32.748 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:32 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: CherryPy Checker: 2024-03-29T04:55:32.748 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:32 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: The Application mounted at '' has an empty config. 2024-03-29T04:55:32.748 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:32 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: 2024-03-29T04:55:32.748 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:32 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: [29/Mar/2024:04:55:32] ENGINE Serving on http://:::9283 2024-03-29T04:55:32.748 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:32 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: [29/Mar/2024:04:55:32] ENGINE Bus STARTED 2024-03-29T04:55:32.830 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:32 smithi181 ceph-mon[29765]: pgmap v12: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:32.830 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:32 smithi181 ceph-mon[29765]: osdmap e207: 8 total, 8 up, 8 in 2024-03-29T04:55:32.830 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:32 smithi181 ceph-mon[29765]: Standby manager daemon x started 2024-03-29T04:55:32.830 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:32 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/310053580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:55:32.830 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:32 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/310053580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:55:32.830 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:32 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/310053580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:55:32.830 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:32 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/310053580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:55:32.831 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:32 smithi181 ceph-mon[41183]: pgmap v12: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:32.831 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:32 smithi181 ceph-mon[41183]: osdmap e207: 8 total, 8 up, 8 in 2024-03-29T04:55:32.831 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:32 smithi181 ceph-mon[41183]: Standby manager daemon x started 2024-03-29T04:55:32.831 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:32 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/310053580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:55:32.831 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:32 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/310053580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:55:32.831 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:32 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/310053580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:55:32.831 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:32 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/310053580' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:55:33.132 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:32 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:55:33.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:33 smithi181 ceph-mon[29765]: osdmap e208: 8 total, 8 up, 8 in 2024-03-29T04:55:33.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:33 smithi181 ceph-mon[29765]: mgrmap e30: y(active, since 7s), standbys: x 2024-03-29T04:55:33.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:33 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:55:33.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:33 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3957359902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:33.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:33 smithi181 ceph-mon[41183]: osdmap e208: 8 total, 8 up, 8 in 2024-03-29T04:55:33.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:33 smithi181 ceph-mon[41183]: mgrmap e30: y(active, since 7s), standbys: x 2024-03-29T04:55:33.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:33 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:55:33.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:33 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3957359902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:33.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:33 smithi187 ceph-mon[33595]: osdmap e208: 8 total, 8 up, 8 in 2024-03-29T04:55:33.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:33 smithi187 ceph-mon[33595]: mgrmap e30: y(active, since 7s), standbys: x 2024-03-29T04:55:33.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:33 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:55:33.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:33 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3957359902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:34.594 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:55:34.611 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 83886079~1 2024-03-29T04:55:34.619 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:55:34.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:34 smithi187 ceph-mon[33595]: pgmap v15: 137 pgs: 137 active+clean; 649 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 377 KiB/s rd, 8.0 MiB/s wr, 455 op/s 2024-03-29T04:55:34.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:34 smithi187 ceph-mon[33595]: osdmap e209: 8 total, 8 up, 8 in 2024-03-29T04:55:34.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:34 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:34.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:34 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:34.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:34 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:34.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:34 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:55:34.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:34 smithi181 ceph-mon[41183]: pgmap v15: 137 pgs: 137 active+clean; 649 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 377 KiB/s rd, 8.0 MiB/s wr, 455 op/s 2024-03-29T04:55:34.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:34 smithi181 ceph-mon[41183]: osdmap e209: 8 total, 8 up, 8 in 2024-03-29T04:55:34.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:34 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:34.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:34 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:34.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:34 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:34.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:34 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:55:34.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:34 smithi181 ceph-mon[29765]: pgmap v15: 137 pgs: 137 active+clean; 649 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 377 KiB/s rd, 8.0 MiB/s wr, 455 op/s 2024-03-29T04:55:34.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:34 smithi181 ceph-mon[29765]: osdmap e209: 8 total, 8 up, 8 in 2024-03-29T04:55:34.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:34 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:34.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:34 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:34.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:34 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:34.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:34 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:55:35.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:35 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[70145]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:55:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:55:35.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:35 smithi187 ceph-mon[33595]: Updating smithi181:/etc/ceph/ceph.conf 2024-03-29T04:55:35.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:35 smithi187 ceph-mon[33595]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:55:35.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:35 smithi187 ceph-mon[33595]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:55:35.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:35 smithi187 ceph-mon[33595]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:55:35.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:35 smithi187 ceph-mon[33595]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:55:35.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:35 smithi187 ceph-mon[33595]: osdmap e210: 8 total, 8 up, 8 in 2024-03-29T04:55:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:35 smithi187 ceph-mon[33595]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:55:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:35 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:35 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:35 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:35 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:35 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:35 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3788920761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:35.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[29765]: Updating smithi181:/etc/ceph/ceph.conf 2024-03-29T04:55:35.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[29765]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:55:35.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[29765]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:55:35.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[29765]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:55:35.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[29765]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:55:35.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[29765]: osdmap e210: 8 total, 8 up, 8 in 2024-03-29T04:55:35.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[29765]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:55:35.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:35.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:35.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:35.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:35.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:35.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3788920761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:35.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[41183]: Updating smithi181:/etc/ceph/ceph.conf 2024-03-29T04:55:35.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[41183]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:55:35.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[41183]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:55:35.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[41183]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:55:35.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[41183]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:55:35.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[41183]: osdmap e210: 8 total, 8 up, 8 in 2024-03-29T04:55:35.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[41183]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:55:35.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:35.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:35.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:35.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:35.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:35.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:35 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3788920761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:36.873 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:36 smithi187 ceph-mon[33595]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:55:36.874 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:36 smithi187 ceph-mon[33595]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:55:36.874 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:36 smithi187 ceph-mon[33595]: pgmap v18: 137 pgs: 137 active+clean; 649 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 377 KiB/s rd, 8.0 MiB/s wr, 455 op/s 2024-03-29T04:55:36.874 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:36 smithi187 ceph-mon[33595]: pgmap v19: 137 pgs: 137 active+clean; 649 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:36.874 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:36 smithi187 ceph-mon[33595]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:55:36.874 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:36 smithi187 ceph-mon[33595]: Reconfiguring daemon prometheus.a on smithi187 2024-03-29T04:55:36.874 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:36 smithi187 ceph-mon[33595]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:55:36.874 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:36 smithi187 ceph-mon[33595]: osdmap e211: 8 total, 8 up, 8 in 2024-03-29T04:55:36.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:36 smithi181 ceph-mon[29765]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:55:36.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:36 smithi181 ceph-mon[29765]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:55:36.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:36 smithi181 ceph-mon[29765]: pgmap v18: 137 pgs: 137 active+clean; 649 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 377 KiB/s rd, 8.0 MiB/s wr, 455 op/s 2024-03-29T04:55:36.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:36 smithi181 ceph-mon[29765]: pgmap v19: 137 pgs: 137 active+clean; 649 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:36.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:36 smithi181 ceph-mon[29765]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:55:36.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:36 smithi181 ceph-mon[29765]: Reconfiguring daemon prometheus.a on smithi187 2024-03-29T04:55:36.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:36 smithi181 ceph-mon[29765]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:55:36.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:36 smithi181 ceph-mon[29765]: osdmap e211: 8 total, 8 up, 8 in 2024-03-29T04:55:36.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:36 smithi181 ceph-mon[41183]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:55:36.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:36 smithi181 ceph-mon[41183]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:55:36.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:36 smithi181 ceph-mon[41183]: pgmap v18: 137 pgs: 137 active+clean; 649 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 377 KiB/s rd, 8.0 MiB/s wr, 455 op/s 2024-03-29T04:55:36.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:36 smithi181 ceph-mon[41183]: pgmap v19: 137 pgs: 137 active+clean; 649 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:36.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:36 smithi181 ceph-mon[41183]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:55:36.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:36 smithi181 ceph-mon[41183]: Reconfiguring daemon prometheus.a on smithi187 2024-03-29T04:55:36.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:36 smithi181 ceph-mon[41183]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:55:36.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:36 smithi181 ceph-mon[41183]: osdmap e211: 8 total, 8 up, 8 in 2024-03-29T04:55:37.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:37 smithi187 ceph-mon[33595]: osdmap e212: 8 total, 8 up, 8 in 2024-03-29T04:55:37.877 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:37 smithi187 ceph-mon[33595]: osdmap e213: 8 total, 8 up, 8 in 2024-03-29T04:55:37.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:37 smithi181 ceph-mon[41183]: osdmap e212: 8 total, 8 up, 8 in 2024-03-29T04:55:37.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:37 smithi181 ceph-mon[41183]: osdmap e213: 8 total, 8 up, 8 in 2024-03-29T04:55:37.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:37 smithi181 ceph-mon[29765]: osdmap e212: 8 total, 8 up, 8 in 2024-03-29T04:55:37.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:37 smithi181 ceph-mon[29765]: osdmap e213: 8 total, 8 up, 8 in 2024-03-29T04:55:38.073 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: after_prepare_snap 2024-03-29T04:55:38.817 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:38 smithi187 ceph-mon[33595]: pgmap v22: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 137 KiB/s rd, 4.6 MiB/s wr, 194 op/s 2024-03-29T04:55:38.817 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:38 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3927115958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:38.864 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:38 smithi181 ceph-mon[41183]: pgmap v22: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 137 KiB/s rd, 4.6 MiB/s wr, 194 op/s 2024-03-29T04:55:38.864 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:38 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3927115958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:38.865 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:38 smithi181 ceph-mon[29765]: pgmap v22: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 137 KiB/s rd, 4.6 MiB/s wr, 194 op/s 2024-03-29T04:55:38.865 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:38 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3927115958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:39.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:38 smithi187 systemd[1]: Stopping Ceph prometheus.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:55:39.538 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:39 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:55:39.192Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:55:39.539 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:39 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:55:39.192Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-29T04:55:39.539 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:39 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:55:39.192Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-29T04:55:39.539 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:39 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:55:39.192Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-29T04:55:39.539 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:39 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:55:39.192Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-29T04:55:39.539 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:39 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:55:39.192Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-29T04:55:39.539 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:39 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:55:39.192Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-29T04:55:39.539 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:39 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:55:39.192Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-29T04:55:39.539 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:39 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:55:39.193Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-29T04:55:39.539 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:39 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:55:39.193Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-29T04:55:39.539 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:39 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:55:39.193Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-29T04:55:39.539 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:39 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[79101]: ts=2024-03-29T04:55:39.193Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-29T04:55:39.539 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:39 smithi187 podman[82849]: 2024-03-29 04:55:39.203421641 +0000 UTC m=+0.136512607 container died b1ce8fda3ac677a95a468e35927931fb0338fa64782c5994a01dc920daa39710 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:55:39.630 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 41943040 2024-03-29T04:55:39.659 INFO:tasks.workunit.client.0.smithi181.stdout:write: * 0~1000 2024-03-29T04:55:39.664 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:55:39.814 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:55:39.880 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:39 smithi187 ceph-mon[33595]: osdmap e214: 8 total, 8 up, 8 in 2024-03-29T04:55:39.880 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:39 smithi187 podman[82849]: 2024-03-29 04:55:39.639575333 +0000 UTC m=+0.572666297 container cleanup b1ce8fda3ac677a95a468e35927931fb0338fa64782c5994a01dc920daa39710 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:55:39.880 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:39 smithi187 bash[82849]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a 2024-03-29T04:55:39.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:39 smithi181 ceph-mon[41183]: osdmap e214: 8 total, 8 up, 8 in 2024-03-29T04:55:39.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:39 smithi181 ceph-mon[29765]: osdmap e214: 8 total, 8 up, 8 in 2024-03-29T04:55:40.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:39 smithi187 podman[82869]: 2024-03-29 04:55:39.874587914 +0000 UTC m=+0.674090853 container remove b1ce8fda3ac677a95a468e35927931fb0338fa64782c5994a01dc920daa39710 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:55:40.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-03-29T04:55:40.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 systemd[1]: Stopped Ceph prometheus.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:40.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@prometheus.a.service: Consumed 1.183s CPU time. 2024-03-29T04:55:40.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 systemd[1]: Starting Ceph prometheus.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:55:40.608 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 podman[82968]: 2024-03-29 04:55:40.219905181 +0000 UTC m=+0.019175218 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-03-29T04:55:40.608 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 podman[82968]: 2024-03-29 04:55:40.375867655 +0000 UTC m=+0.175137747 container create 882b8922d74b4354de1ae38e9c7891149e96c0d90602a271459f5d44cd7b5145 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:55:40.609 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 podman[82968]: 2024-03-29 04:55:40.603239275 +0000 UTC m=+0.402509312 container init 882b8922d74b4354de1ae38e9c7891149e96c0d90602a271459f5d44cd7b5145 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:55:40.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-mon[33595]: pgmap v25: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-03-29T04:55:40.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-mon[33595]: osdmap e215: 8 total, 8 up, 8 in 2024-03-29T04:55:40.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:40.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:40.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3502998847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:40.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 podman[82968]: 2024-03-29 04:55:40.608468779 +0000 UTC m=+0.407738803 container start 882b8922d74b4354de1ae38e9c7891149e96c0d90602a271459f5d44cd7b5145 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:55:40.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.638Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-29T04:55:40.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.638Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-29T04:55:40.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.638Z caller=main.go:570 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi187 (none))" 2024-03-29T04:55:40.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.638Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-29T04:55:40.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.638Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-29T04:55:40.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.641Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-29T04:55:40.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.641Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-29T04:55:40.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.642Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-29T04:55:40.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.642Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-29T04:55:40.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.643Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-29T04:55:40.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.643Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.489µs 2024-03-29T04:55:40.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.643Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-29T04:55:40.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.643Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-03-29T04:55:40.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.660Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-03-29T04:55:40.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.662Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-03-29T04:55:40.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.662Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-03-29T04:55:40.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.662Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=17.314µs wal_replay_duration=19.31555ms wbl_replay_duration=92ns total_replay_duration=19.350152ms 2024-03-29T04:55:40.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.665Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-29T04:55:40.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.665Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-29T04:55:40.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.665Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-29T04:55:40.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.682Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.590695ms db_storage=931ns remote_storage=1.204µs web_handler=431ns query_engine=690ns scrape=409.104µs scrape_sd=77.543µs notify=10.048µs notify_sd=8.103µs rules=15.865105ms tracing=5.168µs 2024-03-29T04:55:40.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.682Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-29T04:55:40.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:40.682Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-29T04:55:40.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 bash[82968]: 882b8922d74b4354de1ae38e9c7891149e96c0d90602a271459f5d44cd7b5145 2024-03-29T04:55:40.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:40 smithi187 systemd[1]: Started Ceph prometheus.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:40.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:40 smithi181 ceph-mon[29765]: pgmap v25: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-03-29T04:55:40.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:40 smithi181 ceph-mon[29765]: osdmap e215: 8 total, 8 up, 8 in 2024-03-29T04:55:40.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:40 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:40.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:40 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:40.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:40 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3502998847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:40.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:40 smithi181 ceph-mon[41183]: pgmap v25: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-03-29T04:55:40.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:40 smithi181 ceph-mon[41183]: osdmap e215: 8 total, 8 up, 8 in 2024-03-29T04:55:40.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:40 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:40.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:40 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:40.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:40 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3502998847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:41.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:41 smithi187 ceph-mon[33595]: osdmap e216: 8 total, 8 up, 8 in 2024-03-29T04:55:41.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:41 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:41.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:41 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:41.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:41 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:55:41.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:41 smithi187 ceph-mon[33595]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:55:41.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:41 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:41.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:41 smithi181 ceph-mon[29765]: osdmap e216: 8 total, 8 up, 8 in 2024-03-29T04:55:41.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:41 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:41.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:41 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:41.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:41 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:55:41.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:41 smithi181 ceph-mon[29765]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:55:41.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:41 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:41.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:41 smithi181 ceph-mon[41183]: osdmap e216: 8 total, 8 up, 8 in 2024-03-29T04:55:41.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:41 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:41.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:41 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:41.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:41 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:55:41.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:41 smithi181 ceph-mon[41183]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:55:41.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:41 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:42.829 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.SnapsSnapAfterPrepare (14200 ms) 2024-03-29T04:55:42.829 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.SnapsSnapExpandAfterPrepare 2024-03-29T04:55:42.859 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:55:42.869 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:55:42.927 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:55:42.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:42 smithi181 ceph-mon[29765]: pgmap v28: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 539 op/s 2024-03-29T04:55:42.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:42 smithi181 ceph-mon[29765]: osdmap e217: 8 total, 8 up, 8 in 2024-03-29T04:55:42.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:42 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:55:42.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:42 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:55:42.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:42 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:42.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:42 smithi181 ceph-mon[29765]: osdmap e218: 8 total, 8 up, 8 in 2024-03-29T04:55:42.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:42 smithi181 ceph-mon[41183]: pgmap v28: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 539 op/s 2024-03-29T04:55:42.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:42 smithi181 ceph-mon[41183]: osdmap e217: 8 total, 8 up, 8 in 2024-03-29T04:55:42.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:42 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:55:42.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:42 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:55:42.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:42 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:42.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:42 smithi181 ceph-mon[41183]: osdmap e218: 8 total, 8 up, 8 in 2024-03-29T04:55:43.010 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:42 smithi187 ceph-mon[33595]: pgmap v28: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 539 op/s 2024-03-29T04:55:43.010 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:42 smithi187 ceph-mon[33595]: osdmap e217: 8 total, 8 up, 8 in 2024-03-29T04:55:43.010 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:42 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:55:43.010 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:42 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:55:43.011 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:42 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:43.011 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:42 smithi187 ceph-mon[33595]: osdmap e218: 8 total, 8 up, 8 in 2024-03-29T04:55:43.011 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:42 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 3. 2024-03-29T04:55:43.011 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:42 smithi187 systemd[1]: Stopped Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:43.011 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:42 smithi187 systemd[1]: Starting Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:55:43.632 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:43 smithi187 podman[83339]: 2024-03-29 04:55:43.363939963 +0000 UTC m=+0.148032354 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:55:43.928 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:43 smithi187 ceph-mon[33595]: Upgrade: Updating mgr.x 2024-03-29T04:55:43.928 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:43 smithi187 ceph-mon[33595]: Deploying daemon mgr.x on smithi187 2024-03-29T04:55:43.928 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:43 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/324361553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:43.928 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:43 smithi187 podman[83339]: 2024-03-29 04:55:43.807147062 +0000 UTC m=+0.591239455 container create a840ee528ef441134e55b770314debeb7bf4ede241521325b0e6e45920203400 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, description=Ceph Grafana Container, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., version=8.5, name=ubi8, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, vcs-type=git, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-03-29T04:55:43.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:43 smithi181 ceph-mon[29765]: Upgrade: Updating mgr.x 2024-03-29T04:55:43.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:43 smithi181 ceph-mon[29765]: Deploying daemon mgr.x on smithi187 2024-03-29T04:55:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:43 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/324361553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:43.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:43 smithi181 ceph-mon[41183]: Upgrade: Updating mgr.x 2024-03-29T04:55:43.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:43 smithi181 ceph-mon[41183]: Deploying daemon mgr.x on smithi187 2024-03-29T04:55:43.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:43 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/324361553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:44.627 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:44 smithi187 podman[83339]: 2024-03-29 04:55:44.504847558 +0000 UTC m=+1.288939951 container init a840ee528ef441134e55b770314debeb7bf4ede241521325b0e6e45920203400 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, version=8.5, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, vcs-type=git, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-03-29T04:55:44.627 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:44 smithi187 podman[83339]: 2024-03-29 04:55:44.513422013 +0000 UTC m=+1.297514404 container start a840ee528ef441134e55b770314debeb7bf4ede241521325b0e6e45920203400 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, architecture=x86_64, name=ubi8, distribution-scope=public, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, io.openshift.expose-services=, io.buildah.version=1.32.0, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-03-29T04:55:44.660 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 8388609~8388609 2024-03-29T04:55:44.683 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 0~1000 2024-03-29T04:55:44.684 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 1010~1000 2024-03-29T04:55:44.684 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap2 2024-03-29T04:55:44.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:44 smithi187 ceph-mon[33595]: pgmap v31: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 539 op/s 2024-03-29T04:55:44.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:44 smithi187 ceph-mon[33595]: osdmap e219: 8 total, 8 up, 8 in 2024-03-29T04:55:44.882 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:44 smithi187 bash[83339]: a840ee528ef441134e55b770314debeb7bf4ede241521325b0e6e45920203400 2024-03-29T04:55:44.882 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:44 smithi187 systemd[1]: Started Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:44.882 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:44 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[83382]: logger=settings t=2024-03-29T04:55:44.678454543Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:55:44.883 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:44 smithi187 podman[83462]: 2024-03-29 04:55:44.713585458 +0000 UTC m=+0.026744866 container died a840ee528ef441134e55b770314debeb7bf4ede241521325b0e6e45920203400 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.buildah.version=1.32.0, release=239.1651231664, version=8.5, vendor=Red Hat, Inc., io.openshift.expose-services=) 2024-03-29T04:55:44.883 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:44 smithi187 systemd[1]: Stopping Ceph mgr.x for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:55:44.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:44 smithi181 ceph-mon[29765]: pgmap v31: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 539 op/s 2024-03-29T04:55:44.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:44 smithi181 ceph-mon[29765]: osdmap e219: 8 total, 8 up, 8 in 2024-03-29T04:55:44.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:44 smithi181 ceph-mon[41183]: pgmap v31: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 539 op/s 2024-03-29T04:55:44.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:44 smithi181 ceph-mon[41183]: osdmap e219: 8 total, 8 up, 8 in 2024-03-29T04:55:45.509 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:45 smithi187 podman[83462]: 2024-03-29 04:55:45.249689579 +0000 UTC m=+0.562849024 container remove a840ee528ef441134e55b770314debeb7bf4ede241521325b0e6e45920203400 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, com.redhat.component=ubi8-container, description=Ceph Grafana Container, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, version=8.5, vendor=Red Hat, Inc., io.openshift.expose-services=, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8) 2024-03-29T04:55:45.509 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:45 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:55:45.809 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:45 smithi187 ceph-mon[33595]: osdmap e220: 8 total, 8 up, 8 in 2024-03-29T04:55:45.809 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:45 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3432175397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:45.809 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:45 smithi187 podman[83512]: 2024-03-29 04:55:45.503376033 +0000 UTC m=+0.160607388 container died eac04768500a0680bb79bd8936ea2f17a56516e37c0122105850f88811b4babf (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-03-29T04:55:45.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:45 smithi181 ceph-mon[29765]: osdmap e220: 8 total, 8 up, 8 in 2024-03-29T04:55:45.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:45 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/3432175397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:45.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:45 smithi181 ceph-mon[41183]: osdmap e220: 8 total, 8 up, 8 in 2024-03-29T04:55:45.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:45 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3432175397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:46.117 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:45 smithi187 podman[83512]: 2024-03-29 04:55:45.877713694 +0000 UTC m=+0.534945042 container cleanup eac04768500a0680bb79bd8936ea2f17a56516e37c0122105850f88811b4babf (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-03-29T04:55:46.118 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:45 smithi187 bash[83512]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x 2024-03-29T04:55:46.382 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:46 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:55:46.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:46 smithi187 podman[83537]: 2024-03-29 04:55:46.112335617 +0000 UTC m=+0.609438078 container remove eac04768500a0680bb79bd8936ea2f17a56516e37c0122105850f88811b4babf (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, RELEASE=squid-8da2362, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:55:46.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:46 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:55:46.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:46 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.x.service: Failed with result 'exit-code'. 2024-03-29T04:55:46.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:46 smithi187 systemd[1]: Stopped Ceph mgr.x for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:46.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:46 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.x.service: Consumed 27.116s CPU time. 2024-03-29T04:55:46.673 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 8388609~1000 2024-03-29T04:55:46.679 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 16777228~1000 2024-03-29T04:55:46.679 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 167772160 2024-03-29T04:55:46.689 INFO:tasks.workunit.client.0.smithi181.stdout:write: 2 83886079~8388609 2024-03-29T04:55:46.714 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap3 2024-03-29T04:55:46.800 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:46 smithi187 ceph-mon[33595]: pgmap v34: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:46.801 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:46 smithi187 ceph-mon[33595]: osdmap e221: 8 total, 8 up, 8 in 2024-03-29T04:55:46.801 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:46 smithi187 systemd[1]: Starting Ceph mgr.x for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:55:46.801 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:46 smithi187 podman[83682]: 2024-03-29 04:55:46.54318513 +0000 UTC m=+0.018015719 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:55:46.801 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:46 smithi187 podman[83682]: 2024-03-29 04:55:46.663351554 +0000 UTC m=+0.138182126 container create 19bc315720bdf7f7413bb32a7c1da0665e3c61c8b6854f9b33ad5c64f3638fbc (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-03-29T04:55:46.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:46 smithi181 ceph-mon[41183]: pgmap v34: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:46.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:46 smithi181 ceph-mon[41183]: osdmap e221: 8 total, 8 up, 8 in 2024-03-29T04:55:46.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:46 smithi181 ceph-mon[29765]: pgmap v34: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:46.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:46 smithi181 ceph-mon[29765]: osdmap e221: 8 total, 8 up, 8 in 2024-03-29T04:55:47.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:47 smithi187 podman[83682]: 2024-03-29 04:55:47.065867139 +0000 UTC m=+0.540697711 container init 19bc315720bdf7f7413bb32a7c1da0665e3c61c8b6854f9b33ad5c64f3638fbc (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:55:47.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:47 smithi187 podman[83682]: 2024-03-29 04:55:47.072014812 +0000 UTC m=+0.546845377 container start 19bc315720bdf7f7413bb32a7c1da0665e3c61c8b6854f9b33ad5c64f3638fbc (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_CLEAN=True) 2024-03-29T04:55:47.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:47 smithi187 ceph-mgr[83720]: -- 172.21.15.187:0/1050388823 <== mon.1 v2:172.21.15.187:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5640531485a0 con 0x56405311c800 2024-03-29T04:55:47.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:47 smithi187 bash[83682]: 19bc315720bdf7f7413bb32a7c1da0665e3c61c8b6854f9b33ad5c64f3638fbc 2024-03-29T04:55:47.383 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:47 smithi187 systemd[1]: Started Ceph mgr.x for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:47.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:47 smithi181 ceph-mon[29765]: osdmap e222: 8 total, 8 up, 8 in 2024-03-29T04:55:47.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:47 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:47.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:47 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:47.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:47 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:47.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:47 smithi181 ceph-mon[41183]: osdmap e222: 8 total, 8 up, 8 in 2024-03-29T04:55:47.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:47 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:47.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:47 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:47.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:47 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:48.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:47 smithi187 ceph-mon[33595]: osdmap e222: 8 total, 8 up, 8 in 2024-03-29T04:55:48.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:47 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:48.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:47 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:48.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:47 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:48.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:47 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:47.647+0000 7fb339281200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:55:48.133 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:47 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:47.758+0000 7fb339281200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:55:48.671 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:48.441+0000 7fb339281200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:55:48.671 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:48.534+0000 7fb339281200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:55:48.693 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:55:48.716 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 83886079~1 2024-03-29T04:55:48.724 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:55:48.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:48 smithi181 ceph-mon[29765]: pgmap v37: 137 pgs: 137 active+clean; 705 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 141 op/s 2024-03-29T04:55:48.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:48 smithi181 ceph-mon[29765]: osdmap e223: 8 total, 8 up, 8 in 2024-03-29T04:55:48.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:48 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2612290024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:48.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:48 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:48.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:48 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:48.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:48 smithi181 ceph-mon[41183]: pgmap v37: 137 pgs: 137 active+clean; 705 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 141 op/s 2024-03-29T04:55:48.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:48 smithi181 ceph-mon[41183]: osdmap e223: 8 total, 8 up, 8 in 2024-03-29T04:55:48.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:48 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2612290024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:48.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:48 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:48.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:48 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:49.036 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:48 smithi187 ceph-mon[33595]: pgmap v37: 137 pgs: 137 active+clean; 705 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 141 op/s 2024-03-29T04:55:49.036 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:48 smithi187 ceph-mon[33595]: osdmap e223: 8 total, 8 up, 8 in 2024-03-29T04:55:49.036 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:48 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2612290024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:49.036 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:48 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:49.036 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:48 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:49.036 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:48.694+0000 7fb339281200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:55:49.036 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:48.805+0000 7fb339281200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:55:49.037 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:48.889+0000 7fb339281200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:55:49.288 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:49 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:49.170+0000 7fb339281200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:55:49.288 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:49 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:49.281+0000 7fb339281200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:55:49.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:55:48] "GET /metrics HTTP/1.1" 200 38726 "" "Prometheus/2.43.0" 2024-03-29T04:55:49.632 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:49 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:49.369+0000 7fb339281200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:55:49.632 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:49 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:49.463+0000 7fb339281200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:55:49.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:49 smithi181 ceph-mon[41183]: osdmap e224: 8 total, 8 up, 8 in 2024-03-29T04:55:49.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:49 smithi181 ceph-mon[29765]: osdmap e224: 8 total, 8 up, 8 in 2024-03-29T04:55:50.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:49 smithi187 ceph-mon[33595]: osdmap e224: 8 total, 8 up, 8 in 2024-03-29T04:55:50.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:49 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:49.686+0000 7fb339281200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:55:50.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:50 smithi181 ceph-mon[29765]: pgmap v40: 137 pgs: 137 active+clean; 705 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 141 op/s 2024-03-29T04:55:50.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:50 smithi181 ceph-mon[29765]: osdmap e225: 8 total, 8 up, 8 in 2024-03-29T04:55:50.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:50 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2187160859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:50.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:50 smithi181 ceph-mon[41183]: pgmap v40: 137 pgs: 137 active+clean; 705 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 141 op/s 2024-03-29T04:55:50.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:50 smithi181 ceph-mon[41183]: osdmap e225: 8 total, 8 up, 8 in 2024-03-29T04:55:50.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:50 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2187160859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:51.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:50 smithi187 ceph-mon[33595]: pgmap v40: 137 pgs: 137 active+clean; 705 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 141 op/s 2024-03-29T04:55:51.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:50 smithi187 ceph-mon[33595]: osdmap e225: 8 total, 8 up, 8 in 2024-03-29T04:55:51.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:50 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2187160859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:51.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:50 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:50.712+0000 7fb339281200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:55:51.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:50 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:50.868+0000 7fb339281200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:55:51.535 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:51.234+0000 7fb339281200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:55:51.535 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:51.329+0000 7fb339281200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:55:51.535 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:51.529+0000 7fb339281200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:55:51.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:51 smithi187 ceph-mon[33595]: osdmap e226: 8 total, 8 up, 8 in 2024-03-29T04:55:51.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:51.732+0000 7fb339281200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:55:51.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:51 smithi181 ceph-mon[41183]: osdmap e226: 8 total, 8 up, 8 in 2024-03-29T04:55:51.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:51 smithi181 ceph-mon[29765]: osdmap e226: 8 total, 8 up, 8 in 2024-03-29T04:55:52.130 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: after_prepare_snap 2024-03-29T04:55:52.341 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:52.049+0000 7fb339281200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:55:52.598 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:52.334+0000 7fb339281200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:55:52.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:52.592+0000 7fb339281200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:55:52.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:52 smithi187 ceph-mon[33595]: pgmap v43: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-03-29T04:55:52.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:52 smithi187 ceph-mon[33595]: osdmap e227: 8 total, 8 up, 8 in 2024-03-29T04:55:52.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:52 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:52.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:52 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:52.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:52 smithi187 ceph-mon[33595]: osdmap e228: 8 total, 8 up, 8 in 2024-03-29T04:55:52.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:52 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/783051866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:52.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:52 smithi181 ceph-mon[41183]: pgmap v43: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-03-29T04:55:52.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:52 smithi181 ceph-mon[41183]: osdmap e227: 8 total, 8 up, 8 in 2024-03-29T04:55:52.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:52 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:52.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:52 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:52.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:52 smithi181 ceph-mon[41183]: osdmap e228: 8 total, 8 up, 8 in 2024-03-29T04:55:52.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:52 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/783051866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:52.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:52 smithi181 ceph-mon[29765]: pgmap v43: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-03-29T04:55:52.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:52 smithi181 ceph-mon[29765]: osdmap e227: 8 total, 8 up, 8 in 2024-03-29T04:55:52.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:52 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:52.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:52 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:52.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:52 smithi181 ceph-mon[29765]: osdmap e228: 8 total, 8 up, 8 in 2024-03-29T04:55:52.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:52 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/783051866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:53.501 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 167772160 2024-03-29T04:55:53.514 INFO:tasks.workunit.client.0.smithi181.stdout:write: * 83886080~1000 2024-03-29T04:55:53.521 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:55:53.670 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:55:53.858 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:53 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:53.473+0000 7fb339281200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:55:54.108 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:53 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:54.108 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:53 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:54.108 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:53 smithi187 ceph-mon[33595]: pgmap v46: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-03-29T04:55:54.108 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:53 smithi187 ceph-mon[33595]: osdmap e229: 8 total, 8 up, 8 in 2024-03-29T04:55:54.109 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:54 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:54.101+0000 7fb339281200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:55:54.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:53 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:54.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:53 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:54.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:53 smithi181 ceph-mon[29765]: pgmap v46: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-03-29T04:55:54.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:53 smithi181 ceph-mon[29765]: osdmap e229: 8 total, 8 up, 8 in 2024-03-29T04:55:54.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:53 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:54.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:53 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:54.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:53 smithi181 ceph-mon[41183]: pgmap v46: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-03-29T04:55:54.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:53 smithi181 ceph-mon[41183]: osdmap e229: 8 total, 8 up, 8 in 2024-03-29T04:55:54.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:54 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:54.202+0000 7fb339281200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:55:54.826 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:54 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:54.431+0000 7fb339281200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:55:54.826 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:54 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: [29/Mar/2024:04:55:54] ENGINE Bus STARTING 2024-03-29T04:55:54.826 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:54 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: CherryPy Checker: 2024-03-29T04:55:54.826 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:54 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: The Application mounted at '' has an empty config. 2024-03-29T04:55:54.826 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:54 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:55:54.826 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:54 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: [29/Mar/2024:04:55:54] ENGINE Serving on http://:::9283 2024-03-29T04:55:54.826 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:55:54 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: [29/Mar/2024:04:55:54] ENGINE Bus STARTED 2024-03-29T04:55:55.091 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:54 smithi187 ceph-mon[33595]: Standby manager daemon x restarted 2024-03-29T04:55:55.091 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:54 smithi187 ceph-mon[33595]: Standby manager daemon x started 2024-03-29T04:55:55.091 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:54 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/976036082' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:55:55.091 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:54 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/976036082' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:55:55.091 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:54 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/976036082' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:55:55.091 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:54 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/976036082' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:55:55.092 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:54 smithi187 ceph-mon[33595]: osdmap e230: 8 total, 8 up, 8 in 2024-03-29T04:55:55.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:54 smithi181 ceph-mon[41183]: Standby manager daemon x restarted 2024-03-29T04:55:55.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:54 smithi181 ceph-mon[41183]: Standby manager daemon x started 2024-03-29T04:55:55.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:54 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/976036082' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:55:55.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:54 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/976036082' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:55:55.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:54 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/976036082' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:55:55.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:54 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/976036082' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:55:55.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:54 smithi181 ceph-mon[41183]: osdmap e230: 8 total, 8 up, 8 in 2024-03-29T04:55:55.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:54 smithi181 ceph-mon[29765]: Standby manager daemon x restarted 2024-03-29T04:55:55.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:54 smithi181 ceph-mon[29765]: Standby manager daemon x started 2024-03-29T04:55:55.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:54 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/976036082' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:55:55.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:54 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/976036082' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:55:55.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:54 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/976036082' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:55:55.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:54 smithi181 ceph-mon[29765]: from='mgr.? 172.21.15.187:0/976036082' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:55:55.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:54 smithi181 ceph-mon[29765]: osdmap e230: 8 total, 8 up, 8 in 2024-03-29T04:55:56.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:55 smithi187 ceph-mon[33595]: mgrmap e31: y(active, since 29s), standbys: x 2024-03-29T04:55:56.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:55 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/1491420344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:56.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:55 smithi187 ceph-mon[33595]: pgmap v49: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:56.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:55 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:56.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:55 smithi187 ceph-mon[33595]: osdmap e231: 8 total, 8 up, 8 in 2024-03-29T04:55:56.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:55 smithi181 ceph-mon[41183]: mgrmap e31: y(active, since 29s), standbys: x 2024-03-29T04:55:56.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:55 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/1491420344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:56.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:55 smithi181 ceph-mon[41183]: pgmap v49: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:56.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:55 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:56.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:55 smithi181 ceph-mon[41183]: osdmap e231: 8 total, 8 up, 8 in 2024-03-29T04:55:56.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:55 smithi181 ceph-mon[29765]: mgrmap e31: y(active, since 29s), standbys: x 2024-03-29T04:55:56.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:55 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/1491420344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:56.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:55 smithi181 ceph-mon[29765]: pgmap v49: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:55:56.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:55 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:56.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:55 smithi181 ceph-mon[29765]: osdmap e231: 8 total, 8 up, 8 in 2024-03-29T04:55:56.435 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:56 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 4. 2024-03-29T04:55:56.435 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:56 smithi187 systemd[1]: Stopped Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:56.436 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:56 smithi187 systemd[1]: Starting Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:55:56.920 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.SnapsSnapExpandAfterPrepare (14090 ms) 2024-03-29T04:55:56.920 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.Clone 2024-03-29T04:55:56.950 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:55:56.963 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 0~8388609 2024-03-29T04:55:56.998 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:56 smithi187 podman[87068]: 2024-03-29 04:55:56.595184435 +0000 UTC m=+0.018603545 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:55:56.998 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:56 smithi187 podman[87068]: 2024-03-29 04:55:56.724382776 +0000 UTC m=+0.147801872 container create 44bf4f87de84d7f5620f9effb5c8f8d98a0031ad6e6ac2dbc186847ef8751537 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, architecture=x86_64, io.openshift.expose-services=, io.buildah.version=1.32.0, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, release=239.1651231664, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-03-29T04:55:56.998 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:56 smithi187 podman[87068]: 2024-03-29 04:55:56.992274313 +0000 UTC m=+0.415693424 container init 44bf4f87de84d7f5620f9effb5c8f8d98a0031ad6e6ac2dbc186847ef8751537 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, vcs-type=git, com.redhat.component=ubi8-container, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, architecture=x86_64, io.openshift.tags=base rhel8, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, name=ubi8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-29T04:55:57.036 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 16777218~8388609 2024-03-29T04:55:57.091 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap 2024-03-29T04:55:57.382 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:56 smithi187 podman[87068]: 2024-03-29 04:55:56.998749906 +0000 UTC m=+0.422169005 container start 44bf4f87de84d7f5620f9effb5c8f8d98a0031ad6e6ac2dbc186847ef8751537 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , vcs-type=git, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, io.buildah.version=1.32.0, version=8.5, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, vendor=Red Hat, Inc., description=Ceph Grafana Container) 2024-03-29T04:55:57.382 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:57 smithi187 bash[87068]: 44bf4f87de84d7f5620f9effb5c8f8d98a0031ad6e6ac2dbc186847ef8751537 2024-03-29T04:55:57.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:57 smithi187 systemd[1]: Started Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:55:57.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[87114]: logger=settings t=2024-03-29T04:55:57.157635967Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:55:57.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:57 smithi187 podman[87149]: 2024-03-29 04:55:57.188463374 +0000 UTC m=+0.019327029 container died 44bf4f87de84d7f5620f9effb5c8f8d98a0031ad6e6ac2dbc186847ef8751537 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, description=Ceph Grafana Container, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, maintainer=Paul Cuzner ) 2024-03-29T04:55:57.674 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:57.675 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:57.675 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:57.675 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:55:57.675 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: pgmap v51: 137 pgs: 137 active+clean; 749 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 501 op/s 2024-03-29T04:55:57.675 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: pgmap v53: 137 pgs: 137 active+clean; 749 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 51 MiB/s rd, 8.0 MiB/s wr, 668 op/s 2024-03-29T04:55:57.675 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: osdmap e232: 8 total, 8 up, 8 in 2024-03-29T04:55:57.675 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:57.675 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:57.675 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:55:57.675 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: Upgrade: Setting container_image for all mgr 2024-03-29T04:55:57.675 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:57.675 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-29T04:55:57.675 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-03-29T04:55:57.676 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-29T04:55:57.676 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-03-29T04:55:57.676 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:55:57.676 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-03-29T04:55:57.676 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: Upgrade: It appears safe to stop mon.a 2024-03-29T04:55:57.676 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/2226811900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:57.676 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:57.676 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:57.676 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:57.676 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:55:57.676 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: pgmap v51: 137 pgs: 137 active+clean; 749 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 501 op/s 2024-03-29T04:55:57.676 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: pgmap v53: 137 pgs: 137 active+clean; 749 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 51 MiB/s rd, 8.0 MiB/s wr, 668 op/s 2024-03-29T04:55:57.677 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: osdmap e232: 8 total, 8 up, 8 in 2024-03-29T04:55:57.677 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:57.677 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:57.677 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:55:57.677 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: Upgrade: Setting container_image for all mgr 2024-03-29T04:55:57.677 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:57.677 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-29T04:55:57.677 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-03-29T04:55:57.677 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-29T04:55:57.677 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-03-29T04:55:57.677 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:55:57.677 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-03-29T04:55:57.677 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: Upgrade: It appears safe to stop mon.a 2024-03-29T04:55:57.677 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:57 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2226811900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:57.771 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:57.771 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:57.771 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:57.771 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:55:57.771 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: pgmap v51: 137 pgs: 137 active+clean; 749 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 501 op/s 2024-03-29T04:55:57.771 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: pgmap v53: 137 pgs: 137 active+clean; 749 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 51 MiB/s rd, 8.0 MiB/s wr, 668 op/s 2024-03-29T04:55:57.771 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: osdmap e232: 8 total, 8 up, 8 in 2024-03-29T04:55:57.771 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:57.771 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:57.771 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:55:57.772 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: Upgrade: Setting container_image for all mgr 2024-03-29T04:55:57.772 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:57.772 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-29T04:55:57.772 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-03-29T04:55:57.772 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-29T04:55:57.772 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-03-29T04:55:57.772 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:55:57.772 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-03-29T04:55:57.772 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: Upgrade: It appears safe to stop mon.a 2024-03-29T04:55:57.772 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:57 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2226811900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:55:58.133 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:57 smithi187 podman[87149]: 2024-03-29 04:55:57.765056508 +0000 UTC m=+0.595920205 container remove 44bf4f87de84d7f5620f9effb5c8f8d98a0031ad6e6ac2dbc186847ef8751537 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, version=8.5, vcs-type=git, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, release=239.1651231664, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, name=ubi8, architecture=x86_64, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner ) 2024-03-29T04:55:58.133 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:57 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:55:58.133 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:55:58 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:55:58.688 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap 2024-03-29T04:55:58.698 INFO:tasks.workunit.client.0.smithi181.stdout:clone image105 -> image108 2024-03-29T04:55:58.797 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 0~1000 2024-03-29T04:55:58.860 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 1010~1000 2024-03-29T04:55:58.862 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap 2024-03-29T04:55:58.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:58 smithi181 ceph-mon[29765]: osdmap e233: 8 total, 8 up, 8 in 2024-03-29T04:55:58.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:58 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:58.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:58 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:55:58.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:58 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:55:58.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:58 smithi181 ceph-mon[29765]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:58.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:58 smithi181 ceph-mon[41183]: osdmap e233: 8 total, 8 up, 8 in 2024-03-29T04:55:58.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:58 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:58.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:58 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:55:58.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:58 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:55:58.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:58 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:59.132 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:55:58 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:55:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:55:59.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:58 smithi187 ceph-mon[33595]: osdmap e233: 8 total, 8 up, 8 in 2024-03-29T04:55:59.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:58 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' 2024-03-29T04:55:59.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:58 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:55:59.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:58 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:55:59.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:58 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:59.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:55:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:55:58] "GET /metrics HTTP/1.1" 200 38731 "" "Prometheus/2.43.0" 2024-03-29T04:55:59.752 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:59 smithi181 ceph-mon[41183]: Upgrade: Updating mon.a 2024-03-29T04:55:59.752 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:59 smithi181 ceph-mon[41183]: Deploying daemon mon.a on smithi181 2024-03-29T04:55:59.752 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:59 smithi181 ceph-mon[41183]: pgmap v55: 137 pgs: 137 active+clean; 749 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 45 MiB/s rd, 7.1 MiB/s wr, 589 op/s 2024-03-29T04:55:59.752 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:55:59 smithi181 ceph-mon[41183]: osdmap e234: 8 total, 8 up, 8 in 2024-03-29T04:55:59.752 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:59 smithi181 ceph-mon[29765]: Upgrade: Updating mon.a 2024-03-29T04:55:59.752 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:59 smithi181 ceph-mon[29765]: Deploying daemon mon.a on smithi181 2024-03-29T04:55:59.753 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:59 smithi181 ceph-mon[29765]: pgmap v55: 137 pgs: 137 active+clean; 749 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 45 MiB/s rd, 7.1 MiB/s wr, 589 op/s 2024-03-29T04:55:59.753 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:55:59 smithi181 ceph-mon[29765]: osdmap e234: 8 total, 8 up, 8 in 2024-03-29T04:55:59.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:59 smithi187 ceph-mon[33595]: Upgrade: Updating mon.a 2024-03-29T04:55:59.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:59 smithi187 ceph-mon[33595]: Deploying daemon mon.a on smithi181 2024-03-29T04:55:59.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:59 smithi187 ceph-mon[33595]: pgmap v55: 137 pgs: 137 active+clean; 749 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 45 MiB/s rd, 7.1 MiB/s wr, 589 op/s 2024-03-29T04:55:59.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:55:59 smithi187 ceph-mon[33595]: osdmap e234: 8 total, 8 up, 8 in 2024-03-29T04:56:00.704 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap 2024-03-29T04:56:00.714 INFO:tasks.workunit.client.0.smithi181.stdout:clone image108 -> image110 2024-03-29T04:56:00.769 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:00 smithi181 ceph-mon[41183]: osdmap e235: 8 total, 8 up, 8 in 2024-03-29T04:56:00.770 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:00 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/815625995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:56:00.770 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:00 smithi181 systemd[1]: Stopping Ceph mon.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:56:00.770 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:00 smithi181 ceph-mon[29765]: osdmap e235: 8 total, 8 up, 8 in 2024-03-29T04:56:00.770 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:00 smithi181 ceph-mon[29765]: from='client.? 172.21.15.181:0/815625995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:56:00.834 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 1000~1000 2024-03-29T04:56:00.889 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 2010~1000 2024-03-29T04:56:00.891 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:56:01.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:00 smithi187 ceph-mon[33595]: osdmap e235: 8 total, 8 up, 8 in 2024-03-29T04:56:01.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:00 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/815625995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:56:01.134 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:00 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a[29742]: 2024-03-29T04:56:00.762+0000 7fe469e99700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:56:01.135 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:00 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a[29742]: 2024-03-29T04:56:00.762+0000 7fe469e99700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-03-29T04:56:01.135 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:00 smithi181 podman[101747]: 2024-03-29 04:56:00.824282766 +0000 UTC m=+0.187418126 container died 3e3936015cad924909bce492f26f753eb72f818dfd57ce0d65915978f085b479 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-03-29T04:56:01.135 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:00 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:56:00] ENGINE Bus STOPPING 2024-03-29T04:56:01.135 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:00 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:56:00] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:56:01.135 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:00 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:56:00] ENGINE Bus STOPPED 2024-03-29T04:56:01.135 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:00 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:56:00] ENGINE Bus STARTING 2024-03-29T04:56:01.135 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:00 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:56:00] ENGINE Serving on http://:::9283 2024-03-29T04:56:01.135 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:00 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:56:00] ENGINE Bus STARTED 2024-03-29T04:56:01.180 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:56:01.349 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:56:01.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:01 smithi181 podman[101747]: 2024-03-29 04:56:01.337250939 +0000 UTC m=+0.700386305 container cleanup 3e3936015cad924909bce492f26f753eb72f818dfd57ce0d65915978f085b479 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240326, ceph=True, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:56:01.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:01 smithi181 bash[101747]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a 2024-03-29T04:56:01.526 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.Clone (4606 ms) 2024-03-29T04:56:01.526 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.CloneParent 2024-03-29T04:56:01.560 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:56:01.571 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap 2024-03-29T04:56:01.908 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:01 smithi181 podman[101777]: 2024-03-29 04:56:01.580936464 +0000 UTC m=+0.755558560 container remove 3e3936015cad924909bce492f26f753eb72f818dfd57ce0d65915978f085b479 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux ) 2024-03-29T04:56:01.908 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:01 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.a.service: Deactivated successfully. 2024-03-29T04:56:01.908 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:01 smithi181 systemd[1]: Stopped Ceph mon.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:56:01.908 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:01 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.a.service: Consumed 15.102s CPU time. 2024-03-29T04:56:02.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:01 smithi181 systemd[1]: Starting Ceph mon.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:56:02.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 podman[101930]: 2024-03-29 04:56:02.019271045 +0000 UTC m=+0.017605308 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:56:02.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 podman[101930]: 2024-03-29 04:56:02.149202013 +0000 UTC m=+0.147536270 container create 4e5d3325d991c36d51de6dea6c47c4695bb2443f666e6b95e63cfe05ab4e4d94 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-03-29T04:56:02.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 podman[101930]: 2024-03-29 04:56:02.355887323 +0000 UTC m=+0.354221576 container init 4e5d3325d991c36d51de6dea6c47c4695bb2443f666e6b95e63cfe05ab4e4d94 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:56:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 podman[101930]: 2024-03-29 04:56:02.361128593 +0000 UTC m=+0.359462843 container start 4e5d3325d991c36d51de6dea6c47c4695bb2443f666e6b95e63cfe05ab4e4d94 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=) 2024-03-29T04:56:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: set uid:gid to 167:167 (ceph:ceph) 2024-03-29T04:56:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev), process ceph-mon, pid 7 2024-03-29T04:56:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: pidfile_write: ignore empty --pid-file 2024-03-29T04:56:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: load: jerasure load: lrc 2024-03-29T04:56:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: RocksDB version: 7.9.2 2024-03-29T04:56:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Git sha 0 2024-03-29T04:56:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Compile date 2024-03-28 15:35:57 2024-03-29T04:56:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: DB SUMMARY 2024-03-29T04:56:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: DB Session ID: 93H1PZINONF4N1ANS82T 2024-03-29T04:56:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: CURRENT file: CURRENT 2024-03-29T04:56:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: IDENTITY file: IDENTITY 2024-03-29T04:56:02.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1174 Bytes 2024-03-29T04:56:02.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-03-29T04:56:02.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 1927490 ; 2024-03-29T04:56:02.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.error_if_exists: 0 2024-03-29T04:56:02.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.create_if_missing: 0 2024-03-29T04:56:02.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.paranoid_checks: 1 2024-03-29T04:56:02.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-29T04:56:02.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-29T04:56:02.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-29T04:56:02.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.env: 0x562cfa35ec60 2024-03-29T04:56:02.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.fs: PosixFileSystem 2024-03-29T04:56:02.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.info_log: 0x562cfc53dae0 2024-03-29T04:56:02.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_file_opening_threads: 16 2024-03-29T04:56:02.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.statistics: (nil) 2024-03-29T04:56:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.use_fsync: 0 2024-03-29T04:56:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_log_file_size: 0 2024-03-29T04:56:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-29T04:56:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-29T04:56:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.keep_log_file_num: 1000 2024-03-29T04:56:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.recycle_log_file_num: 0 2024-03-29T04:56:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.allow_fallocate: 1 2024-03-29T04:56:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.allow_mmap_reads: 0 2024-03-29T04:56:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.allow_mmap_writes: 0 2024-03-29T04:56:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.use_direct_reads: 0 2024-03-29T04:56:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-29T04:56:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.create_missing_column_families: 0 2024-03-29T04:56:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.db_log_dir: 2024-03-29T04:56:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.wal_dir: 2024-03-29T04:56:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-29T04:56:02.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-29T04:56:02.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-29T04:56:02.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-29T04:56:02.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-29T04:56:02.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-29T04:56:02.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.advise_random_on_open: 1 2024-03-29T04:56:02.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.db_write_buffer_size: 0 2024-03-29T04:56:02.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.write_buffer_manager: 0x562cfc510820 2024-03-29T04:56:02.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-29T04:56:02.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-29T04:56:02.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-29T04:56:02.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.rate_limiter: (nil) 2024-03-29T04:56:02.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-29T04:56:02.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.wal_recovery_mode: 2 2024-03-29T04:56:02.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.enable_thread_tracking: 0 2024-03-29T04:56:02.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.enable_pipelined_write: 0 2024-03-29T04:56:02.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.unordered_write: 0 2024-03-29T04:56:02.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-29T04:56:02.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-29T04:56:02.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-29T04:56:02.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-29T04:56:02.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.row_cache: None 2024-03-29T04:56:02.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.wal_filter: None 2024-03-29T04:56:02.709 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-29T04:56:02.709 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.allow_ingest_behind: 0 2024-03-29T04:56:02.709 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.two_write_queues: 0 2024-03-29T04:56:02.709 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.manual_wal_flush: 0 2024-03-29T04:56:02.709 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.wal_compression: 0 2024-03-29T04:56:02.709 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.atomic_flush: 0 2024-03-29T04:56:02.709 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-29T04:56:02.709 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-29T04:56:02.709 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-29T04:56:02.709 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.log_readahead_size: 0 2024-03-29T04:56:02.709 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-29T04:56:02.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.best_efforts_recovery: 0 2024-03-29T04:56:02.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-29T04:56:02.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-29T04:56:02.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.allow_data_in_errors: 0 2024-03-29T04:56:02.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.db_host_id: __hostname__ 2024-03-29T04:56:02.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-29T04:56:02.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_background_jobs: 2 2024-03-29T04:56:02.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_background_compactions: -1 2024-03-29T04:56:02.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_subcompactions: 1 2024-03-29T04:56:02.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-29T04:56:02.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-29T04:56:02.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-29T04:56:02.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_total_wal_size: 0 2024-03-29T04:56:02.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-29T04:56:02.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-29T04:56:02.711 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-29T04:56:02.711 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-29T04:56:02.711 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_open_files: -1 2024-03-29T04:56:02.711 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.bytes_per_sync: 0 2024-03-29T04:56:02.711 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-29T04:56:02.711 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-29T04:56:02.711 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compaction_readahead_size: 0 2024-03-29T04:56:02.711 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_background_flushes: -1 2024-03-29T04:56:02.711 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Compression algorithms supported: 2024-03-29T04:56:02.711 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-29T04:56:02.711 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: kZSTD supported: 0 2024-03-29T04:56:02.711 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: kXpressCompression supported: 0 2024-03-29T04:56:02.711 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: kLZ4HCCompression supported: 1 2024-03-29T04:56:02.712 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: kZlibCompression supported: 1 2024-03-29T04:56:02.712 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: kSnappyCompression supported: 1 2024-03-29T04:56:02.712 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: kLZ4Compression supported: 1 2024-03-29T04:56:02.712 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: kBZip2Compression supported: 0 2024-03-29T04:56:02.712 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-29T04:56:02.712 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-29T04:56:02.712 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-03-29T04:56:02.712 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-29T04:56:02.712 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-29T04:56:02.712 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.merge_operator: 2024-03-29T04:56:02.712 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compaction_filter: None 2024-03-29T04:56:02.712 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compaction_filter_factory: None 2024-03-29T04:56:02.712 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.sst_partitioner_factory: None 2024-03-29T04:56:02.712 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-29T04:56:02.712 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-29T04:56:02.712 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562cfc53dbe0) 2024-03-29T04:56:02.713 INFO:journalctl@ceph.mon.a.smithi181.stdout: cache_index_and_filter_blocks: 1 2024-03-29T04:56:02.713 INFO:journalctl@ceph.mon.a.smithi181.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-29T04:56:02.713 INFO:journalctl@ceph.mon.a.smithi181.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-29T04:56:02.713 INFO:journalctl@ceph.mon.a.smithi181.stdout: pin_top_level_index_and_filter: 1 2024-03-29T04:56:02.713 INFO:journalctl@ceph.mon.a.smithi181.stdout: index_type: 0 2024-03-29T04:56:02.713 INFO:journalctl@ceph.mon.a.smithi181.stdout: data_block_index_type: 0 2024-03-29T04:56:02.713 INFO:journalctl@ceph.mon.a.smithi181.stdout: index_shortening: 1 2024-03-29T04:56:02.713 INFO:journalctl@ceph.mon.a.smithi181.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-29T04:56:02.713 INFO:journalctl@ceph.mon.a.smithi181.stdout: checksum: 4 2024-03-29T04:56:02.713 INFO:journalctl@ceph.mon.a.smithi181.stdout: no_block_cache: 0 2024-03-29T04:56:02.713 INFO:journalctl@ceph.mon.a.smithi181.stdout: block_cache: 0x562cfc541090 2024-03-29T04:56:02.713 INFO:journalctl@ceph.mon.a.smithi181.stdout: block_cache_name: BinnedLRUCache 2024-03-29T04:56:02.713 INFO:journalctl@ceph.mon.a.smithi181.stdout: block_cache_options: 2024-03-29T04:56:02.713 INFO:journalctl@ceph.mon.a.smithi181.stdout: capacity : 536870912 2024-03-29T04:56:02.713 INFO:journalctl@ceph.mon.a.smithi181.stdout: num_shard_bits : 4 2024-03-29T04:56:02.714 INFO:journalctl@ceph.mon.a.smithi181.stdout: strict_capacity_limit : 0 2024-03-29T04:56:02.714 INFO:journalctl@ceph.mon.a.smithi181.stdout: high_pri_pool_ratio: 0.000 2024-03-29T04:56:02.714 INFO:journalctl@ceph.mon.a.smithi181.stdout: block_cache_compressed: (nil) 2024-03-29T04:56:02.714 INFO:journalctl@ceph.mon.a.smithi181.stdout: persistent_cache: (nil) 2024-03-29T04:56:02.714 INFO:journalctl@ceph.mon.a.smithi181.stdout: block_size: 4096 2024-03-29T04:56:02.714 INFO:journalctl@ceph.mon.a.smithi181.stdout: block_size_deviation: 10 2024-03-29T04:56:02.714 INFO:journalctl@ceph.mon.a.smithi181.stdout: block_restart_interval: 16 2024-03-29T04:56:02.714 INFO:journalctl@ceph.mon.a.smithi181.stdout: index_block_restart_interval: 1 2024-03-29T04:56:02.714 INFO:journalctl@ceph.mon.a.smithi181.stdout: metadata_block_size: 4096 2024-03-29T04:56:02.714 INFO:journalctl@ceph.mon.a.smithi181.stdout: partition_filters: 0 2024-03-29T04:56:02.714 INFO:journalctl@ceph.mon.a.smithi181.stdout: use_delta_encoding: 1 2024-03-29T04:56:02.714 INFO:journalctl@ceph.mon.a.smithi181.stdout: filter_policy: bloomfilter 2024-03-29T04:56:02.714 INFO:journalctl@ceph.mon.a.smithi181.stdout: whole_key_filtering: 1 2024-03-29T04:56:02.714 INFO:journalctl@ceph.mon.a.smithi181.stdout: verify_compression: 0 2024-03-29T04:56:02.714 INFO:journalctl@ceph.mon.a.smithi181.stdout: read_amp_bytes_per_bit: 0 2024-03-29T04:56:02.715 INFO:journalctl@ceph.mon.a.smithi181.stdout: format_version: 5 2024-03-29T04:56:02.715 INFO:journalctl@ceph.mon.a.smithi181.stdout: enable_index_compression: 1 2024-03-29T04:56:02.715 INFO:journalctl@ceph.mon.a.smithi181.stdout: block_align: 0 2024-03-29T04:56:02.715 INFO:journalctl@ceph.mon.a.smithi181.stdout: max_auto_readahead_size: 262144 2024-03-29T04:56:02.715 INFO:journalctl@ceph.mon.a.smithi181.stdout: prepopulate_block_cache: 0 2024-03-29T04:56:02.715 INFO:journalctl@ceph.mon.a.smithi181.stdout: initial_auto_readahead_size: 8192 2024-03-29T04:56:02.715 INFO:journalctl@ceph.mon.a.smithi181.stdout: num_file_reads_for_auto_readahead: 2 2024-03-29T04:56:02.715 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.write_buffer_size: 33554432 2024-03-29T04:56:02.715 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_write_buffer_number: 2 2024-03-29T04:56:02.715 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compression: NoCompression 2024-03-29T04:56:02.716 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.bottommost_compression: Disabled 2024-03-29T04:56:02.716 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.prefix_extractor: nullptr 2024-03-29T04:56:02.716 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-29T04:56:02.716 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.num_levels: 7 2024-03-29T04:56:02.716 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-29T04:56:02.716 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-29T04:56:02.716 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-29T04:56:02.716 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-29T04:56:02.716 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-29T04:56:02.716 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-29T04:56:02.716 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-29T04:56:02.716 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:56:02.716 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-29T04:56:02.716 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-29T04:56:02.716 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:56:02.717 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:56:02.717 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-29T04:56:02.717 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compression_opts.level: 32767 2024-03-29T04:56:02.717 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compression_opts.strategy: 0 2024-03-29T04:56:02.717 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-29T04:56:02.717 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:56:02.717 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:56:02.717 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-29T04:56:02.717 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compression_opts.enabled: false 2024-03-29T04:56:02.717 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:56:02.717 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-29T04:56:02.717 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-29T04:56:02.717 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-29T04:56:02.717 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.target_file_size_base: 67108864 2024-03-29T04:56:02.717 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-29T04:56:02.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-29T04:56:02.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-29T04:56:02.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-29T04:56:02.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-29T04:56:02.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-29T04:56:02.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-29T04:56:02.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-29T04:56:02.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-29T04:56:02.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-29T04:56:02.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-29T04:56:02.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-29T04:56:02.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-29T04:56:02.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-29T04:56:02.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.arena_block_size: 1048576 2024-03-29T04:56:02.719 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-29T04:56:02.719 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-29T04:56:02.719 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.disable_auto_compactions: 0 2024-03-29T04:56:02.719 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-29T04:56:02.719 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-29T04:56:02.719 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-29T04:56:02.719 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-29T04:56:02.719 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-29T04:56:02.719 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-29T04:56:02.719 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-29T04:56:02.719 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-29T04:56:02.719 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-29T04:56:02.719 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-29T04:56:02.719 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.table_properties_collectors: 2024-03-29T04:56:02.719 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.inplace_update_support: 0 2024-03-29T04:56:02.720 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-29T04:56:02.720 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-29T04:56:02.720 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-29T04:56:02.720 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-29T04:56:02.720 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.bloom_locality: 0 2024-03-29T04:56:02.720 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.max_successive_merges: 0 2024-03-29T04:56:02.720 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-29T04:56:02.720 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.paranoid_file_checks: 0 2024-03-29T04:56:02.720 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.force_consistency_checks: 1 2024-03-29T04:56:02.720 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.report_bg_io_stats: 0 2024-03-29T04:56:02.720 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.ttl: 2592000 2024-03-29T04:56:02.720 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-29T04:56:02.720 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-29T04:56:02.720 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-29T04:56:02.720 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.enable_blob_files: false 2024-03-29T04:56:02.720 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.min_blob_size: 0 2024-03-29T04:56:02.721 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.blob_file_size: 268435456 2024-03-29T04:56:02.721 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-29T04:56:02.721 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-29T04:56:02.721 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-29T04:56:02.721 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-29T04:56:02.721 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-29T04:56:02.721 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.blob_file_starting_level: 0 2024-03-29T04:56:02.721 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-29T04:56:02.721 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-03-29T04:56:02.721 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:56:02.721 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12822, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-29T04:56:02.721 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-03-29T04:56:02.721 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 23916bf6-3b53-4ca2-89d1-ad25f90d246d 2024-03-29T04:56:02.721 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711688162406282, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-03-29T04:56:02.721 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-03-29T04:56:02.722 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711688162412348, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 1786790, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12823, "largest_seqno": 13808, "table_properties": {"data_size": 1781098, "index_size": 3525, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1157, "raw_key_size": 10414, "raw_average_key_size": 23, "raw_value_size": 1770991, "raw_average_value_size": 4052, "num_data_blocks": 156, "num_entries": 437, "num_filter_entries": 437, "num_deletions": 6, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1711688162, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "23916bf6-3b53-4ca2-89d1-ad25f90d246d", "db_session_id": "93H1PZINONF4N1ANS82T", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-03-29T04:56:02.722 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711688162412449, "job": 1, "event": "recovery_finished"} 2024-03-29T04:56:02.722 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-03-29T04:56:02.722 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:56:02.722 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-29T04:56:02.722 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x562cfc632000 2024-03-29T04:56:02.722 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: DB pointer 0x562cfc61c000 2024-03-29T04:56:02.722 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-29T04:56:02.722 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-29T04:56:02.722 INFO:journalctl@ceph.mon.a.smithi181.stdout: ** DB Stats ** 2024-03-29T04:56:02.722 INFO:journalctl@ceph.mon.a.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:56:02.722 INFO:journalctl@ceph.mon.a.smithi181.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-29T04:56:02.722 INFO:journalctl@ceph.mon.a.smithi181.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:56:02.722 INFO:journalctl@ceph.mon.a.smithi181.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:56:02.722 INFO:journalctl@ceph.mon.a.smithi181.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-29T04:56:02.723 INFO:journalctl@ceph.mon.a.smithi181.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:56:02.723 INFO:journalctl@ceph.mon.a.smithi181.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:56:02.723 INFO:journalctl@ceph.mon.a.smithi181.stdout: 2024-03-29T04:56:02.723 INFO:journalctl@ceph.mon.a.smithi181.stdout: ** Compaction Stats [default] ** 2024-03-29T04:56:02.723 INFO:journalctl@ceph.mon.a.smithi181.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-29T04:56:02.723 INFO:journalctl@ceph.mon.a.smithi181.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-29T04:56:02.723 INFO:journalctl@ceph.mon.a.smithi181.stdout: L0 1/0 1.70 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 525.3 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-03-29T04:56:02.723 INFO:journalctl@ceph.mon.a.smithi181.stdout: L6 1/0 8.35 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-29T04:56:02.723 INFO:journalctl@ceph.mon.a.smithi181.stdout: Sum 2/0 10.06 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 525.3 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-03-29T04:56:02.723 INFO:journalctl@ceph.mon.a.smithi181.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 525.3 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-03-29T04:56:02.723 INFO:journalctl@ceph.mon.a.smithi181.stdout: 2024-03-29T04:56:02.723 INFO:journalctl@ceph.mon.a.smithi181.stdout: ** Compaction Stats [default] ** 2024-03-29T04:56:02.723 INFO:journalctl@ceph.mon.a.smithi181.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-29T04:56:02.723 INFO:journalctl@ceph.mon.a.smithi181.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:56:02.724 INFO:journalctl@ceph.mon.a.smithi181.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 525.3 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-03-29T04:56:02.724 INFO:journalctl@ceph.mon.a.smithi181.stdout: 2024-03-29T04:56:02.724 INFO:journalctl@ceph.mon.a.smithi181.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-29T04:56:02.724 INFO:journalctl@ceph.mon.a.smithi181.stdout: 2024-03-29T04:56:02.724 INFO:journalctl@ceph.mon.a.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:56:02.724 INFO:journalctl@ceph.mon.a.smithi181.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-03-29T04:56:02.724 INFO:journalctl@ceph.mon.a.smithi181.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:56:02.724 INFO:journalctl@ceph.mon.a.smithi181.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:56:02.724 INFO:journalctl@ceph.mon.a.smithi181.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:56:02.724 INFO:journalctl@ceph.mon.a.smithi181.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:56:02.724 INFO:journalctl@ceph.mon.a.smithi181.stdout: Cumulative compaction: 0.00 GB write, 81.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:56:02.724 INFO:journalctl@ceph.mon.a.smithi181.stdout: Interval compaction: 0.00 GB write, 81.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:56:02.724 INFO:journalctl@ceph.mon.a.smithi181.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:56:02.724 INFO:journalctl@ceph.mon.a.smithi181.stdout: Block cache BinnedLRUCache@0x562cfc541090#7 capacity: 512.00 MB usage: 5.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.8e-05 secs_since: 0 2024-03-29T04:56:02.724 INFO:journalctl@ceph.mon.a.smithi181.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-03-29T04:56:02.725 INFO:journalctl@ceph.mon.a.smithi181.stdout: 2024-03-29T04:56:02.725 INFO:journalctl@ceph.mon.a.smithi181.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:56:02.725 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: starting mon.a rank 0 at public addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] at bind addrs [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:56:02.725 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: mon.a@-1(???) e3 preinit fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:56:02.725 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: mon.a@-1(???).mds e9 new map 2024-03-29T04:56:02.725 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: mon.a@-1(???).mds e9 print_map 2024-03-29T04:56:02.725 INFO:journalctl@ceph.mon.a.smithi181.stdout: e9 2024-03-29T04:56:02.725 INFO:journalctl@ceph.mon.a.smithi181.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-29T04:56:02.725 INFO:journalctl@ceph.mon.a.smithi181.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:56:02.725 INFO:journalctl@ceph.mon.a.smithi181.stdout: legacy client fscid: 1 2024-03-29T04:56:02.725 INFO:journalctl@ceph.mon.a.smithi181.stdout: 2024-03-29T04:56:02.725 INFO:journalctl@ceph.mon.a.smithi181.stdout: Filesystem 'foo' (1) 2024-03-29T04:56:02.725 INFO:journalctl@ceph.mon.a.smithi181.stdout: fs_name foo 2024-03-29T04:56:02.725 INFO:journalctl@ceph.mon.a.smithi181.stdout: epoch 8 2024-03-29T04:56:02.725 INFO:journalctl@ceph.mon.a.smithi181.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-29T04:56:02.726 INFO:journalctl@ceph.mon.a.smithi181.stdout: created 2024-03-29T04:51:50.886956+0000 2024-03-29T04:56:02.726 INFO:journalctl@ceph.mon.a.smithi181.stdout: modified 2024-03-29T04:52:07.243441+0000 2024-03-29T04:56:02.726 INFO:journalctl@ceph.mon.a.smithi181.stdout: tableserver 0 2024-03-29T04:56:02.726 INFO:journalctl@ceph.mon.a.smithi181.stdout: root 0 2024-03-29T04:56:02.726 INFO:journalctl@ceph.mon.a.smithi181.stdout: session_timeout 60 2024-03-29T04:56:02.726 INFO:journalctl@ceph.mon.a.smithi181.stdout: session_autoclose 300 2024-03-29T04:56:02.726 INFO:journalctl@ceph.mon.a.smithi181.stdout: max_file_size 1099511627776 2024-03-29T04:56:02.726 INFO:journalctl@ceph.mon.a.smithi181.stdout: max_xattr_size 65536 2024-03-29T04:56:02.726 INFO:journalctl@ceph.mon.a.smithi181.stdout: required_client_features {} 2024-03-29T04:56:02.726 INFO:journalctl@ceph.mon.a.smithi181.stdout: last_failure 0 2024-03-29T04:56:02.726 INFO:journalctl@ceph.mon.a.smithi181.stdout: last_failure_osd_epoch 0 2024-03-29T04:56:02.726 INFO:journalctl@ceph.mon.a.smithi181.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:56:02.726 INFO:journalctl@ceph.mon.a.smithi181.stdout: max_mds 1 2024-03-29T04:56:02.726 INFO:journalctl@ceph.mon.a.smithi181.stdout: in 0 2024-03-29T04:56:02.727 INFO:journalctl@ceph.mon.a.smithi181.stdout: up {0=24560} 2024-03-29T04:56:02.727 INFO:journalctl@ceph.mon.a.smithi181.stdout: failed 2024-03-29T04:56:02.727 INFO:journalctl@ceph.mon.a.smithi181.stdout: damaged 2024-03-29T04:56:02.727 INFO:journalctl@ceph.mon.a.smithi181.stdout: stopped 2024-03-29T04:56:02.727 INFO:journalctl@ceph.mon.a.smithi181.stdout: data_pools [4] 2024-03-29T04:56:02.727 INFO:journalctl@ceph.mon.a.smithi181.stdout: metadata_pool 3 2024-03-29T04:56:02.727 INFO:journalctl@ceph.mon.a.smithi181.stdout: inline_data disabled 2024-03-29T04:56:02.727 INFO:journalctl@ceph.mon.a.smithi181.stdout: balancer 2024-03-29T04:56:02.727 INFO:journalctl@ceph.mon.a.smithi181.stdout: bal_rank_mask -1 2024-03-29T04:56:02.727 INFO:journalctl@ceph.mon.a.smithi181.stdout: standby_count_wanted 1 2024-03-29T04:56:02.727 INFO:journalctl@ceph.mon.a.smithi181.stdout: qdb_cluster leader: 0 members: 2024-03-29T04:56:02.727 INFO:journalctl@ceph.mon.a.smithi181.stdout: [mds.foo.smithi181.pflrwm{0:24560} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.181:6834/2778467308,v1:172.21.15.181:6835/2778467308] compat {c=[1],r=[1],i=[7ff]}] 2024-03-29T04:56:02.727 INFO:journalctl@ceph.mon.a.smithi181.stdout: 2024-03-29T04:56:02.727 INFO:journalctl@ceph.mon.a.smithi181.stdout: 2024-03-29T04:56:02.727 INFO:journalctl@ceph.mon.a.smithi181.stdout: Standby daemons: 2024-03-29T04:56:02.727 INFO:journalctl@ceph.mon.a.smithi181.stdout: 2024-03-29T04:56:02.728 INFO:journalctl@ceph.mon.a.smithi181.stdout: [mds.foo.smithi187.vcsnqx{-1:24511} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.187:6832/1383464008,v1:172.21.15.187:6833/1383464008] compat {c=[1],r=[1],i=[7ff]}] 2024-03-29T04:56:02.728 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: mon.a@-1(???).osd e236 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-29T04:56:02.728 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: mon.a@-1(???).osd e236 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:56:02.728 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: mon.a@-1(???).osd e236 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:56:02.728 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: mon.a@-1(???).osd e236 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:56:02.728 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 ceph-mon[101970]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2024-03-29T04:56:03.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 bash[101930]: 4e5d3325d991c36d51de6dea6c47c4695bb2443f666e6b95e63cfe05ab4e4d94 2024-03-29T04:56:03.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:02 smithi181 systemd[1]: Started Ceph mon.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:56:04.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ignoring --setuser ceph since I am not root 2024-03-29T04:56:04.211 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ignoring --setgroup ceph since I am not root 2024-03-29T04:56:04.211 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mgr[92292]: -- 172.21.15.181:0/3012867405 <== mon.0 v2:172.21.15.181:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563b417285a0 con 0x563b416fcc00 2024-03-29T04:56:04.212 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:56:04.212 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:56:04.212 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:56:04.212 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3978945957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:56:04.212 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: pgmap v60: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 10 MiB/s wr, 433 op/s 2024-03-29T04:56:04.212 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: mon.a calling monitor election 2024-03-29T04:56:04.212 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:56:04.212 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: monmap epoch 3 2024-03-29T04:56:04.212 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:56:04.212 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: last_changed 2024-03-29T04:46:30.517496+0000 2024-03-29T04:56:04.212 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: created 2024-03-29T04:44:46.123593+0000 2024-03-29T04:56:04.212 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: min_mon_release 17 (quincy) 2024-03-29T04:56:04.212 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: election_strategy: 1 2024-03-29T04:56:04.212 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: 0: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.a 2024-03-29T04:56:04.213 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: 1: [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon.b 2024-03-29T04:56:04.213 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: 2: [v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0] mon.c 2024-03-29T04:56:04.213 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:56:04.213 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: osdmap e236: 8 total, 8 up, 8 in 2024-03-29T04:56:04.213 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: mgrmap e31: y(active, since 38s), standbys: x 2024-03-29T04:56:04.213 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-03-29T04:56:04.213 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:56:04.213 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: daemon grafana.a on smithi187 is in unknown state 2024-03-29T04:56:04.213 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-29T04:56:04.213 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: application not enabled on pool 'test-librbd-smithi181-86707-9' 2024-03-29T04:56:04.213 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-29T04:56:04.213 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: from='mgr.25160 ' entity='' 2024-03-29T04:56:04.213 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[101970]: mgrmap e32: y(active, since 38s), standbys: x 2024-03-29T04:56:04.213 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:56:04.214 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:56:04.214 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:56:04.214 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3978945957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:56:04.214 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: pgmap v60: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 10 MiB/s wr, 433 op/s 2024-03-29T04:56:04.214 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: mon.a calling monitor election 2024-03-29T04:56:04.214 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:56:04.214 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: monmap epoch 3 2024-03-29T04:56:04.214 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:56:04.214 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: last_changed 2024-03-29T04:46:30.517496+0000 2024-03-29T04:56:04.214 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: created 2024-03-29T04:44:46.123593+0000 2024-03-29T04:56:04.214 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: min_mon_release 17 (quincy) 2024-03-29T04:56:04.214 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: election_strategy: 1 2024-03-29T04:56:04.214 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: 0: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.a 2024-03-29T04:56:04.214 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: 1: [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon.b 2024-03-29T04:56:04.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: 2: [v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0] mon.c 2024-03-29T04:56:04.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:56:04.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: osdmap e236: 8 total, 8 up, 8 in 2024-03-29T04:56:04.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: mgrmap e31: y(active, since 38s), standbys: x 2024-03-29T04:56:04.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-03-29T04:56:04.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:56:04.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: daemon grafana.a on smithi187 is in unknown state 2024-03-29T04:56:04.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-29T04:56:04.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: application not enabled on pool 'test-librbd-smithi181-86707-9' 2024-03-29T04:56:04.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-29T04:56:04.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: from='mgr.25160 ' entity='' 2024-03-29T04:56:04.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-mon[41183]: mgrmap e32: y(active, since 38s), standbys: x 2024-03-29T04:56:04.215 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:03 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:03.897+0000 7fced2a08700 -1 mon.c@2(peon).osd e237 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:04.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: ignoring --setuser ceph since I am not root 2024-03-29T04:56:04.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: ignoring --setgroup ceph since I am not root 2024-03-29T04:56:04.382 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mgr[83720]: -- 172.21.15.187:0/1670316844 <== mon.1 v2:172.21.15.187:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56194d4fa5a0 con 0x56194d4cec00 2024-03-29T04:56:04.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:56:04.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:56:04.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: from='mgr.25160 172.21.15.181:0/4250892087' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:56:04.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3978945957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:56:04.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: pgmap v60: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 10 MiB/s wr, 433 op/s 2024-03-29T04:56:04.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: mon.a calling monitor election 2024-03-29T04:56:04.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:56:04.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: monmap epoch 3 2024-03-29T04:56:04.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:56:04.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: last_changed 2024-03-29T04:46:30.517496+0000 2024-03-29T04:56:04.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: created 2024-03-29T04:44:46.123593+0000 2024-03-29T04:56:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: min_mon_release 17 (quincy) 2024-03-29T04:56:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: election_strategy: 1 2024-03-29T04:56:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: 0: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.a 2024-03-29T04:56:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: 1: [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon.b 2024-03-29T04:56:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: 2: [v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0] mon.c 2024-03-29T04:56:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:56:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: osdmap e236: 8 total, 8 up, 8 in 2024-03-29T04:56:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: mgrmap e31: y(active, since 38s), standbys: x 2024-03-29T04:56:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-03-29T04:56:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:56:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: daemon grafana.a on smithi187 is in unknown state 2024-03-29T04:56:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-29T04:56:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: application not enabled on pool 'test-librbd-smithi181-86707-9' 2024-03-29T04:56:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-29T04:56:04.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: from='mgr.25160 ' entity='' 2024-03-29T04:56:04.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-mon[33595]: mgrmap e32: y(active, since 38s), standbys: x 2024-03-29T04:56:04.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:03.895+0000 7f1d52566700 -1 mon.b@1(peon).osd e237 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:04.602 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 1 2024-03-29T04:56:04.702 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:04 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:04.435+0000 7f67bf1f9200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:56:04.702 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:04 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:04.531+0000 7f67bf1f9200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:56:04.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:04.411+0000 7f58c89e9200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:56:04.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:04.504+0000 7f58c89e9200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:56:04.932 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap 2024-03-29T04:56:04.942 INFO:tasks.workunit.client.0.smithi181.stdout:clone image111 -> image114 2024-03-29T04:56:05.047 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:56:05.166 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:04 smithi187 ceph-mon[33595]: osdmap e237: 8 total, 8 up, 8 in 2024-03-29T04:56:05.167 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:04 smithi187 ceph-mon[33595]: failed to encode map e237 with expected crc 2024-03-29T04:56:05.167 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:04 smithi187 ceph-mon[33595]: failed to encode map e237 with expected crc 2024-03-29T04:56:05.167 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:04 smithi187 ceph-mon[33595]: failed to encode map e237 with expected crc 2024-03-29T04:56:05.167 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:04 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/122846953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:56:05.167 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:04.902+0000 7f1d52566700 -1 mon.b@1(peon).osd e238 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:05.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:04 smithi181 ceph-mon[41183]: osdmap e237: 8 total, 8 up, 8 in 2024-03-29T04:56:05.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:04 smithi181 ceph-mon[41183]: failed to encode map e237 with expected crc 2024-03-29T04:56:05.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:04 smithi181 ceph-mon[41183]: failed to encode map e237 with expected crc 2024-03-29T04:56:05.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:04 smithi181 ceph-mon[41183]: failed to encode map e237 with expected crc 2024-03-29T04:56:05.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:04 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/122846953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:56:05.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:04 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:04.902+0000 7fced2a08700 -1 mon.c@2(peon).osd e238 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:05.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:04 smithi181 ceph-mon[101970]: osdmap e237: 8 total, 8 up, 8 in 2024-03-29T04:56:05.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:04 smithi181 ceph-mon[101970]: failed to encode map e237 with expected crc 2024-03-29T04:56:05.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:04 smithi181 ceph-mon[101970]: failed to encode map e237 with expected crc 2024-03-29T04:56:05.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:04 smithi181 ceph-mon[101970]: failed to encode map e237 with expected crc 2024-03-29T04:56:05.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:04 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/122846953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:56:05.222 INFO:tasks.workunit.client.0.smithi181.stdout:migration_abort 2024-03-29T04:56:05.366 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.CloneParent (3841 ms) 2024-03-29T04:56:05.366 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.CloneUpdateAfterPrepare 2024-03-29T04:56:05.398 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:56:05.410 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 0~10 2024-03-29T04:56:05.419 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap 2024-03-29T04:56:05.419 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:05 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:05.160+0000 7f58c89e9200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:56:05.419 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:05 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:05.258+0000 7f58c89e9200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:56:05.419 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:05 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:05.413+0000 7f58c89e9200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:56:05.484 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T04:56:05.541 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:05.263+0000 7f67bf1f9200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:56:05.541 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:05.381+0000 7f67bf1f9200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:56:05.764 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:05.862 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:05 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:05.509+0000 7f58c89e9200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:56:05.863 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:05 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:05.592+0000 7f58c89e9200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:56:05.863 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:05 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:05.856+0000 7f58c89e9200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:56:05.874 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:05.534+0000 7f67bf1f9200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:56:05.874 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:05.637+0000 7f67bf1f9200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:56:05.874 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:05.737+0000 7f67bf1f9200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:56:06.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:05 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:05.970+0000 7f58c89e9200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:56:06.132 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:06 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:06.056+0000 7f58c89e9200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:56:06.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:05 smithi187 ceph-mon[33595]: osdmap e238: 8 total, 8 up, 8 in 2024-03-29T04:56:06.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:05 smithi187 ceph-mon[33595]: failed to encode map e238 with expected crc 2024-03-29T04:56:06.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:05 smithi187 ceph-mon[33595]: failed to encode map e238 with expected crc 2024-03-29T04:56:06.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:05 smithi187 ceph-mon[33595]: failed to encode map e238 with expected crc 2024-03-29T04:56:06.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:05 smithi187 ceph-mon[33595]: failed to encode map e238 with expected crc 2024-03-29T04:56:06.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:05 smithi187 ceph-mon[33595]: failed to encode map e238 with expected crc 2024-03-29T04:56:06.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:05 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:05.908+0000 7f1d52566700 -1 mon.b@1(peon).osd e239 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:06.135 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:06 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:06.033+0000 7f67bf1f9200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:56:06.135 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-mon[101970]: osdmap e238: 8 total, 8 up, 8 in 2024-03-29T04:56:06.135 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-mon[101970]: failed to encode map e238 with expected crc 2024-03-29T04:56:06.135 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-mon[101970]: failed to encode map e238 with expected crc 2024-03-29T04:56:06.135 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-mon[101970]: failed to encode map e238 with expected crc 2024-03-29T04:56:06.135 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-mon[101970]: failed to encode map e238 with expected crc 2024-03-29T04:56:06.136 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-mon[101970]: failed to encode map e238 with expected crc 2024-03-29T04:56:06.136 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-mon[41183]: osdmap e238: 8 total, 8 up, 8 in 2024-03-29T04:56:06.136 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-mon[41183]: failed to encode map e238 with expected crc 2024-03-29T04:56:06.136 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-mon[41183]: failed to encode map e238 with expected crc 2024-03-29T04:56:06.136 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-mon[41183]: failed to encode map e238 with expected crc 2024-03-29T04:56:06.136 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-mon[41183]: failed to encode map e238 with expected crc 2024-03-29T04:56:06.136 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-mon[41183]: failed to encode map e238 with expected crc 2024-03-29T04:56:06.136 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:05 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:05.908+0000 7fced2a08700 -1 mon.c@2(peon).osd e239 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:06.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:06 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:06.149+0000 7f67bf1f9200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:56:06.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:06 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:06.241+0000 7f67bf1f9200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:56:06.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:06 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:06.338+0000 7f67bf1f9200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:56:06.632 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:06 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:06.152+0000 7f58c89e9200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:56:06.632 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:06 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:06.328+0000 7f58c89e9200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:56:06.915 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:06 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:06.520+0000 7f67bf1f9200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:56:06.941 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap 2024-03-29T04:56:06.951 INFO:tasks.workunit.client.0.smithi181.stdout:clone image115 -> image118 2024-03-29T04:56:07.055 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:56:07.166 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:06 smithi181 ceph-mon[101970]: osdmap e239: 8 total, 8 up, 8 in 2024-03-29T04:56:07.166 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:06 smithi181 ceph-mon[101970]: failed to encode map e239 with expected crc 2024-03-29T04:56:07.166 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:06 smithi181 ceph-mon[101970]: failed to encode map e239 with expected crc 2024-03-29T04:56:07.166 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:06 smithi181 ceph-mon[101970]: failed to encode map e239 with expected crc 2024-03-29T04:56:07.167 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:06 smithi181 ceph-mon[101970]: failed to encode map e239 with expected crc 2024-03-29T04:56:07.167 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:06 smithi181 ceph-mon[41183]: osdmap e239: 8 total, 8 up, 8 in 2024-03-29T04:56:07.167 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:06 smithi181 ceph-mon[41183]: failed to encode map e239 with expected crc 2024-03-29T04:56:07.167 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:06 smithi181 ceph-mon[41183]: failed to encode map e239 with expected crc 2024-03-29T04:56:07.167 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:06 smithi181 ceph-mon[41183]: failed to encode map e239 with expected crc 2024-03-29T04:56:07.167 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:06 smithi181 ceph-mon[41183]: failed to encode map e239 with expected crc 2024-03-29T04:56:07.167 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:06 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:06.915+0000 7fced2a08700 -1 mon.c@2(peon).osd e240 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:07.221 INFO:tasks.workunit.client.0.smithi181.stdout:write: Y 0~1 2024-03-29T04:56:07.254 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:56:07.336 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:06 smithi187 ceph-mon[33595]: osdmap e239: 8 total, 8 up, 8 in 2024-03-29T04:56:07.336 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:06 smithi187 ceph-mon[33595]: failed to encode map e239 with expected crc 2024-03-29T04:56:07.336 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:06 smithi187 ceph-mon[33595]: failed to encode map e239 with expected crc 2024-03-29T04:56:07.336 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:06 smithi187 ceph-mon[33595]: failed to encode map e239 with expected crc 2024-03-29T04:56:07.336 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:06 smithi187 ceph-mon[33595]: failed to encode map e239 with expected crc 2024-03-29T04:56:07.336 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:06 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:06.914+0000 7f1d52566700 -1 mon.b@1(peon).osd e240 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:07.337 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:56:07.461 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.CloneUpdateAfterPrepare (2095 ms) 2024-03-29T04:56:07.461 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.TriggerAssertSnapcSeq 2024-03-29T04:56:07.499 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:56:07.510 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 41943040~10 2024-03-29T04:56:07.518 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:56:07.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:07 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:07.422+0000 7f1d52566700 -1 mon.b@1(peon).osd e241 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:07.632 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:07 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:07.329+0000 7f58c89e9200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:56:07.632 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:07 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:07.484+0000 7f58c89e9200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:56:07.702 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:07.604+0000 7f67bf1f9200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:56:07.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:07.422+0000 7fced2a08700 -1 mon.c@2(peon).osd e241 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:08.088 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:07.739+0000 7f67bf1f9200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:56:08.088 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[41183]: osdmap e240: 8 total, 8 up, 8 in 2024-03-29T04:56:08.089 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[41183]: failed to encode map e240 with expected crc 2024-03-29T04:56:08.089 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[41183]: failed to encode map e240 with expected crc 2024-03-29T04:56:08.089 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[41183]: failed to encode map e240 with expected crc 2024-03-29T04:56:08.089 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[41183]: failed to encode map e240 with expected crc 2024-03-29T04:56:08.089 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[41183]: osdmap e241: 8 total, 8 up, 8 in 2024-03-29T04:56:08.089 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[41183]: failed to encode map e241 with expected crc 2024-03-29T04:56:08.089 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[41183]: failed to encode map e241 with expected crc 2024-03-29T04:56:08.089 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[41183]: failed to encode map e241 with expected crc 2024-03-29T04:56:08.089 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[101970]: osdmap e240: 8 total, 8 up, 8 in 2024-03-29T04:56:08.089 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[101970]: failed to encode map e240 with expected crc 2024-03-29T04:56:08.089 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[101970]: failed to encode map e240 with expected crc 2024-03-29T04:56:08.089 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[101970]: failed to encode map e240 with expected crc 2024-03-29T04:56:08.089 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[101970]: failed to encode map e240 with expected crc 2024-03-29T04:56:08.090 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[101970]: osdmap e241: 8 total, 8 up, 8 in 2024-03-29T04:56:08.090 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[101970]: failed to encode map e241 with expected crc 2024-03-29T04:56:08.090 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[101970]: failed to encode map e241 with expected crc 2024-03-29T04:56:08.090 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:07 smithi181 ceph-mon[101970]: failed to encode map e241 with expected crc 2024-03-29T04:56:08.119 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:07 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:07.825+0000 7f58c89e9200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:56:08.119 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:07 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:07.915+0000 7f58c89e9200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:56:08.120 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:07 smithi187 ceph-mon[33595]: osdmap e240: 8 total, 8 up, 8 in 2024-03-29T04:56:08.120 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:07 smithi187 ceph-mon[33595]: failed to encode map e240 with expected crc 2024-03-29T04:56:08.120 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:07 smithi187 ceph-mon[33595]: failed to encode map e240 with expected crc 2024-03-29T04:56:08.120 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:07 smithi187 ceph-mon[33595]: failed to encode map e240 with expected crc 2024-03-29T04:56:08.120 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:07 smithi187 ceph-mon[33595]: failed to encode map e240 with expected crc 2024-03-29T04:56:08.120 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:07 smithi187 ceph-mon[33595]: osdmap e241: 8 total, 8 up, 8 in 2024-03-29T04:56:08.120 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:07 smithi187 ceph-mon[33595]: failed to encode map e241 with expected crc 2024-03-29T04:56:08.120 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:07 smithi187 ceph-mon[33595]: failed to encode map e241 with expected crc 2024-03-29T04:56:08.120 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:07 smithi187 ceph-mon[33595]: failed to encode map e241 with expected crc 2024-03-29T04:56:08.370 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:08 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 5. 2024-03-29T04:56:08.370 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:08 smithi187 systemd[1]: Stopped Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:56:08.370 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:08 smithi187 systemd[1]: Starting Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:56:08.371 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:08.113+0000 7f58c89e9200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:56:08.371 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:08.328+0000 7f58c89e9200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:56:08.378 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:08.081+0000 7f67bf1f9200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:56:08.378 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:08.174+0000 7f67bf1f9200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:56:08.627 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:08.423+0000 7f1d52566700 -1 mon.b@1(peon).osd e242 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:08.628 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:08 smithi187 podman[87321]: 2024-03-29 04:56:08.266270497 +0000 UTC m=+0.017681384 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:56:08.702 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:08.371+0000 7f67bf1f9200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:56:08.702 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:08.569+0000 7f67bf1f9200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:56:08.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:08.425+0000 7fced2a08700 -1 mon.c@2(peon).osd e242 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:08.882 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:56:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:56:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:56:08.883 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:08.621+0000 7f58c89e9200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:56:08.883 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:08 smithi187 podman[87321]: 2024-03-29 04:56:08.682557814 +0000 UTC m=+0.433968702 container create 6c3d80b40904436c7fad2bdb7017be96433f49fd1da49059e149f91c9d641897 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.tags=base rhel8, vcs-type=git, release=239.1651231664, version=8.5, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , name=ubi8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-29T04:56:09.133 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:08.862+0000 7f67bf1f9200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:56:09.134 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:08 smithi181 ceph-mon[41183]: failed to encode map e241 with expected crc 2024-03-29T04:56:09.134 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:08 smithi181 ceph-mon[41183]: failed to encode map e241 with expected crc 2024-03-29T04:56:09.134 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:08 smithi181 ceph-mon[41183]: failed to encode map e242 with expected crc 2024-03-29T04:56:09.134 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:08 smithi181 ceph-mon[41183]: osdmap e242: 8 total, 8 up, 8 in 2024-03-29T04:56:09.134 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:08 smithi181 ceph-mon[41183]: failed to encode map e242 with expected crc 2024-03-29T04:56:09.134 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:08 smithi181 ceph-mon[101970]: failed to encode map e241 with expected crc 2024-03-29T04:56:09.134 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:08 smithi181 ceph-mon[101970]: failed to encode map e241 with expected crc 2024-03-29T04:56:09.134 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:08 smithi181 ceph-mon[101970]: failed to encode map e242 with expected crc 2024-03-29T04:56:09.135 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:08 smithi181 ceph-mon[101970]: osdmap e242: 8 total, 8 up, 8 in 2024-03-29T04:56:09.135 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:08 smithi181 ceph-mon[101970]: failed to encode map e242 with expected crc 2024-03-29T04:56:09.157 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:08 smithi187 ceph-mon[33595]: failed to encode map e241 with expected crc 2024-03-29T04:56:09.157 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:08 smithi187 ceph-mon[33595]: failed to encode map e241 with expected crc 2024-03-29T04:56:09.157 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:08 smithi187 ceph-mon[33595]: failed to encode map e242 with expected crc 2024-03-29T04:56:09.157 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:08 smithi187 ceph-mon[33595]: osdmap e242: 8 total, 8 up, 8 in 2024-03-29T04:56:09.157 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:08 smithi187 ceph-mon[33595]: failed to encode map e242 with expected crc 2024-03-29T04:56:09.158 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:08.892+0000 7f58c89e9200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:56:09.158 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:09 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:09.141+0000 7f58c89e9200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:56:09.158 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:09 smithi187 podman[87321]: 2024-03-29 04:56:09.043669174 +0000 UTC m=+0.795080075 container init 6c3d80b40904436c7fad2bdb7017be96433f49fd1da49059e149f91c9d641897 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, io.openshift.tags=base rhel8, vcs-type=git, vendor=Red Hat, Inc., name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, version=8.5, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, distribution-scope=public, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container) 2024-03-29T04:56:09.158 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:09 smithi187 podman[87321]: 2024-03-29 04:56:09.051137528 +0000 UTC m=+0.802548415 container start 6c3d80b40904436c7fad2bdb7017be96433f49fd1da49059e149f91c9d641897 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, distribution-scope=public, version=8.5) 2024-03-29T04:56:09.432 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:09 smithi187 bash[87321]: 6c3d80b40904436c7fad2bdb7017be96433f49fd1da49059e149f91c9d641897 2024-03-29T04:56:09.433 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:09 smithi187 systemd[1]: Started Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:56:09.433 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:09 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[87341]: logger=settings t=2024-03-29T04:56:09.203830013Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:56:09.433 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:09 smithi187 podman[87381]: 2024-03-29 04:56:09.229699035 +0000 UTC m=+0.018873458 container died 6c3d80b40904436c7fad2bdb7017be96433f49fd1da49059e149f91c9d641897 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, vendor=Red Hat, Inc., release=239.1651231664, version=8.5, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.openshift.expose-services=, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-03-29T04:56:09.434 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:09 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:09.127+0000 7f67bf1f9200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:56:09.434 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:09 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:09.369+0000 7f67bf1f9200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:56:09.450 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 41943041~10 2024-03-29T04:56:09.455 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:56:09.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:09 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:09.427+0000 7fced2a08700 -1 mon.c@2(peon).osd e243 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:09.819 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:09 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:09.426+0000 7f1d52566700 -1 mon.b@1(peon).osd e243 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:10.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:09 smithi187 ceph-mon[33595]: failed to encode map e242 with expected crc 2024-03-29T04:56:10.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:09 smithi187 ceph-mon[33595]: failed to encode map e242 with expected crc 2024-03-29T04:56:10.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:09 smithi187 ceph-mon[33595]: failed to encode map e242 with expected crc 2024-03-29T04:56:10.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:09 smithi187 ceph-mon[33595]: osdmap e243: 8 total, 8 up, 8 in 2024-03-29T04:56:10.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:09 smithi187 ceph-mon[33595]: failed to encode map e243 with expected crc 2024-03-29T04:56:10.133 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:10.003+0000 7f58c89e9200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:56:10.133 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:09 smithi187 podman[87381]: 2024-03-29 04:56:09.923114408 +0000 UTC m=+0.712288819 container remove 6c3d80b40904436c7fad2bdb7017be96433f49fd1da49059e149f91c9d641897 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, description=Ceph Grafana Container, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, distribution-scope=public, name=ubi8, vcs-type=git, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-03-29T04:56:10.133 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:09 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:56:10.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:09 smithi181 ceph-mon[101970]: failed to encode map e242 with expected crc 2024-03-29T04:56:10.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:09 smithi181 ceph-mon[101970]: failed to encode map e242 with expected crc 2024-03-29T04:56:10.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:09 smithi181 ceph-mon[101970]: failed to encode map e242 with expected crc 2024-03-29T04:56:10.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:09 smithi181 ceph-mon[101970]: osdmap e243: 8 total, 8 up, 8 in 2024-03-29T04:56:10.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:09 smithi181 ceph-mon[101970]: failed to encode map e243 with expected crc 2024-03-29T04:56:10.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:09 smithi181 ceph-mon[41183]: failed to encode map e242 with expected crc 2024-03-29T04:56:10.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:09 smithi181 ceph-mon[41183]: failed to encode map e242 with expected crc 2024-03-29T04:56:10.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:09 smithi181 ceph-mon[41183]: failed to encode map e242 with expected crc 2024-03-29T04:56:10.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:09 smithi181 ceph-mon[41183]: osdmap e243: 8 total, 8 up, 8 in 2024-03-29T04:56:10.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:09 smithi181 ceph-mon[41183]: failed to encode map e243 with expected crc 2024-03-29T04:56:10.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:10.192+0000 7f67bf1f9200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:56:10.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:10.427+0000 7fced2a08700 -1 mon.c@2(peon).osd e244 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:10.579 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 41943042~10 2024-03-29T04:56:10.594 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap2 2024-03-29T04:56:10.621 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:10.427+0000 7f1d52566700 -1 mon.b@1(peon).osd e244 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:10.621 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:10 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:56:10.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:10.614+0000 7f58c89e9200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:56:10.882 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:10.716+0000 7f58c89e9200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:56:11.130 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:10.799+0000 7f67bf1f9200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:56:11.130 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:10.899+0000 7f67bf1f9200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:56:11.131 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-mon[101970]: failed to encode map e243 with expected crc 2024-03-29T04:56:11.131 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-mon[101970]: failed to encode map e243 with expected crc 2024-03-29T04:56:11.131 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-mon[101970]: failed to encode map e243 with expected crc 2024-03-29T04:56:11.131 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-mon[101970]: osdmap e244: 8 total, 8 up, 8 in 2024-03-29T04:56:11.131 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-mon[101970]: failed to encode map e244 with expected crc 2024-03-29T04:56:11.131 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-mon[101970]: failed to encode map e244 with expected crc 2024-03-29T04:56:11.131 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-mon[101970]: failed to encode map e244 with expected crc 2024-03-29T04:56:11.131 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-mon[41183]: failed to encode map e243 with expected crc 2024-03-29T04:56:11.131 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-mon[41183]: failed to encode map e243 with expected crc 2024-03-29T04:56:11.131 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-mon[41183]: failed to encode map e243 with expected crc 2024-03-29T04:56:11.131 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-mon[41183]: osdmap e244: 8 total, 8 up, 8 in 2024-03-29T04:56:11.132 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-mon[41183]: failed to encode map e244 with expected crc 2024-03-29T04:56:11.132 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-mon[41183]: failed to encode map e244 with expected crc 2024-03-29T04:56:11.132 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:10 smithi181 ceph-mon[41183]: failed to encode map e244 with expected crc 2024-03-29T04:56:11.270 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:10 smithi187 ceph-mon[33595]: failed to encode map e243 with expected crc 2024-03-29T04:56:11.271 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:10 smithi187 ceph-mon[33595]: failed to encode map e243 with expected crc 2024-03-29T04:56:11.271 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:10 smithi187 ceph-mon[33595]: failed to encode map e243 with expected crc 2024-03-29T04:56:11.271 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:10 smithi187 ceph-mon[33595]: osdmap e244: 8 total, 8 up, 8 in 2024-03-29T04:56:11.271 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:10 smithi187 ceph-mon[33595]: failed to encode map e244 with expected crc 2024-03-29T04:56:11.271 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:10 smithi187 ceph-mon[33595]: failed to encode map e244 with expected crc 2024-03-29T04:56:11.271 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:10 smithi187 ceph-mon[33595]: failed to encode map e244 with expected crc 2024-03-29T04:56:11.271 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:11.129+0000 7f1d52566700 -1 mon.b@1(peon).osd e245 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:11.271 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:10.937+0000 7f58c89e9200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:56:11.271 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: [29/Mar/2024:04:56:10] ENGINE Bus STARTING 2024-03-29T04:56:11.271 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: CherryPy Checker: 2024-03-29T04:56:11.271 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: The Application mounted at '' has an empty config. 2024-03-29T04:56:11.271 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: 2024-03-29T04:56:11.272 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: [29/Mar/2024:04:56:11] ENGINE Serving on http://:::9283 2024-03-29T04:56:11.272 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x[83697]: [29/Mar/2024:04:56:11] ENGINE Bus STARTED 2024-03-29T04:56:11.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:11.128+0000 7fced2a08700 -1 mon.c@2(peon).osd e245 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:11.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:11.123+0000 7f67bf1f9200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:56:11.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:56:11] ENGINE Bus STARTING 2024-03-29T04:56:11.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:11.262+0000 7f67205d3700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:56:11.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:11.262+0000 7f67205d3700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:56:11.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:11.262+0000 7f67205d3700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:56:11.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:11.262+0000 7f67205d3700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:56:11.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:11.262+0000 7f67205d3700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:56:11.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:11.263+0000 7f671c5cb700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:56:11.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:11.263+0000 7f671c5cb700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:56:11.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:11.263+0000 7f671c5cb700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:56:11.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:11.263+0000 7f671c5cb700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:56:11.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:11.263+0000 7f671c5cb700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:56:11.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: CherryPy Checker: 2024-03-29T04:56:11.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: The Application mounted at '' has an empty config. 2024-03-29T04:56:11.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:11.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:56:11] ENGINE Serving on http://:::9283 2024-03-29T04:56:11.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:56:11] ENGINE Bus STARTED 2024-03-29T04:56:12.161 INFO:teuthology.orchestra.run.smithi181.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T04:56:12.161 INFO:teuthology.orchestra.run.smithi181.stdout:alertmanager.a smithi181 *:9093,9094 running (83s) 42s ago 5m 18.4M - 0.25.0 c8568f914cd2 69e0ad67b57f 2024-03-29T04:56:12.161 INFO:teuthology.orchestra.run.smithi181.stdout:grafana.a smithi187 *:3000 unknown 20s ago 5m - - 2024-03-29T04:56:12.161 INFO:teuthology.orchestra.run.smithi181.stdout:mds.foo.smithi181.pflrwm smithi181 running (4m) 42s ago 4m 14.6M - 17.2.7-789-g4213e17c cf22eb8683be 0f200e33dce6 2024-03-29T04:56:12.161 INFO:teuthology.orchestra.run.smithi181.stdout:mds.foo.smithi187.vcsnqx smithi187 running (4m) 20s ago 4m 16.0M - 17.2.7-789-g4213e17c cf22eb8683be 245621ed14ed 2024-03-29T04:56:12.161 INFO:teuthology.orchestra.run.smithi181.stdout:mgr.x smithi187 *:8443,9283,8765 running (25s) 20s ago 9m 67.8M - 19.0.0-1959-g8da23624 54112f57da44 19bc315720bd 2024-03-29T04:56:12.161 INFO:teuthology.orchestra.run.smithi181.stdout:mgr.y smithi181 *:8443,9283,8765 running (56s) 42s ago 11m 478M - 19.0.0-1959-g8da23624 54112f57da44 b07d84f4244a 2024-03-29T04:56:12.161 INFO:teuthology.orchestra.run.smithi181.stdout:mon.a smithi181 running (11m) 42s ago 11m 54.7M 2048M 17.2.7-789-g4213e17c cf22eb8683be 3e3936015cad 2024-03-29T04:56:12.162 INFO:teuthology.orchestra.run.smithi181.stdout:mon.b smithi187 running (9m) 20s ago 9m 49.7M 2048M 17.2.7-789-g4213e17c cf22eb8683be a376bfeefb23 2024-03-29T04:56:12.162 INFO:teuthology.orchestra.run.smithi181.stdout:mon.c smithi181 running (9m) 42s ago 9m 44.6M 2048M 17.2.7-789-g4213e17c cf22eb8683be 16379529f880 2024-03-29T04:56:12.162 INFO:teuthology.orchestra.run.smithi181.stdout:node-exporter.a smithi181 *:9100 running (81s) 42s ago 4m 7629k - 1.5.0 0da6a335fe13 4029d029600b 2024-03-29T04:56:12.162 INFO:teuthology.orchestra.run.smithi181.stdout:node-exporter.b smithi187 *:9100 running (62s) 20s ago 4m 8330k - 1.5.0 0da6a335fe13 3894f6e97d55 2024-03-29T04:56:12.162 INFO:teuthology.orchestra.run.smithi181.stdout:osd.0 smithi181 running (9m) 42s ago 9m 114M 2480M 17.2.7-789-g4213e17c cf22eb8683be ae426d29a1a8 2024-03-29T04:56:12.162 INFO:teuthology.orchestra.run.smithi181.stdout:osd.1 smithi181 running (8m) 42s ago 8m 96.4M 2480M 17.2.7-789-g4213e17c cf22eb8683be 7ed1966c65cf 2024-03-29T04:56:12.162 INFO:teuthology.orchestra.run.smithi181.stdout:osd.2 smithi181 running (8m) 42s ago 8m 86.6M 2480M 17.2.7-789-g4213e17c cf22eb8683be 9c61fb0d684c 2024-03-29T04:56:12.162 INFO:teuthology.orchestra.run.smithi181.stdout:osd.3 smithi181 running (7m) 42s ago 7m 91.5M 2480M 17.2.7-789-g4213e17c cf22eb8683be 6b0bf471cfcb 2024-03-29T04:56:12.162 INFO:teuthology.orchestra.run.smithi181.stdout:osd.4 smithi187 running (7m) 20s ago 7m 109M 2480M 17.2.7-789-g4213e17c cf22eb8683be 42698769ef84 2024-03-29T04:56:12.162 INFO:teuthology.orchestra.run.smithi181.stdout:osd.5 smithi187 running (7m) 20s ago 7m 95.6M 2480M 17.2.7-789-g4213e17c cf22eb8683be 9599ff85a9e7 2024-03-29T04:56:12.162 INFO:teuthology.orchestra.run.smithi181.stdout:osd.6 smithi187 running (6m) 20s ago 6m 111M 2480M 17.2.7-789-g4213e17c cf22eb8683be c3287470c1fa 2024-03-29T04:56:12.162 INFO:teuthology.orchestra.run.smithi181.stdout:osd.7 smithi187 running (6m) 20s ago 6m 97.1M 2480M 17.2.7-789-g4213e17c cf22eb8683be 89fb8ccf8705 2024-03-29T04:56:12.162 INFO:teuthology.orchestra.run.smithi181.stdout:prometheus.a smithi187 *:9095 running (31s) 20s ago 5m 43.9M - 2.43.0 a07b618ecd1d 882b8922d74b 2024-03-29T04:56:12.178 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 41943040 2024-03-29T04:56:12.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:56:12.139+0000 7f67753a3700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-03-29T04:56:12.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: failed to encode map e244 with expected crc 2024-03-29T04:56:12.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: Standby manager daemon x restarted 2024-03-29T04:56:12.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: Standby manager daemon x started 2024-03-29T04:56:12.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.? 172.21.15.187:0/1598439137' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:56:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.? 172.21.15.187:0/1598439137' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:56:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.? 172.21.15.187:0/1598439137' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:56:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.? 172.21.15.187:0/1598439137' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:56:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: Active manager daemon y restarted 2024-03-29T04:56:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: Activating manager daemon y 2024-03-29T04:56:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: osdmap e245: 8 total, 8 up, 8 in 2024-03-29T04:56:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: mgrmap e33: y(active, starting, since 0.00532876s), standbys: x 2024-03-29T04:56:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: failed to encode map e245 with expected crc 2024-03-29T04:56:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: failed to encode map e245 with expected crc 2024-03-29T04:56:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: failed to encode map e245 with expected crc 2024-03-29T04:56:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:56:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:56:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:56:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi187.vcsnqx"}]: dispatch 2024-03-29T04:56:12.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.pflrwm"}]: dispatch 2024-03-29T04:56:12.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:56:12.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:56:12.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:56:12.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:56:12.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:56:12.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:56:12.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:56:12.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:56:12.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:56:12.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:56:12.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:56:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: failed to encode map e244 with expected crc 2024-03-29T04:56:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: Standby manager daemon x restarted 2024-03-29T04:56:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: Standby manager daemon x started 2024-03-29T04:56:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/1598439137' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:56:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/1598439137' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:56:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/1598439137' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:56:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.? 172.21.15.187:0/1598439137' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:56:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: Active manager daemon y restarted 2024-03-29T04:56:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: Activating manager daemon y 2024-03-29T04:56:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: osdmap e245: 8 total, 8 up, 8 in 2024-03-29T04:56:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: mgrmap e33: y(active, starting, since 0.00532876s), standbys: x 2024-03-29T04:56:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: failed to encode map e245 with expected crc 2024-03-29T04:56:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: failed to encode map e245 with expected crc 2024-03-29T04:56:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: failed to encode map e245 with expected crc 2024-03-29T04:56:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:56:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:56:12.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:56:12.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi187.vcsnqx"}]: dispatch 2024-03-29T04:56:12.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.pflrwm"}]: dispatch 2024-03-29T04:56:12.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:56:12.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:56:12.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:56:12.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:56:12.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:56:12.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:56:12.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:56:12.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:56:12.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:56:12.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:56:12.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:56:12.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:56:12.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:56:12.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: Manager daemon y is now available 2024-03-29T04:56:12.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:56:12.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:12.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:56:12.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:56:12.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:56:12.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:56:12.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:56:12.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:56:12.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: Manager daemon y is now available 2024-03-29T04:56:12.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:56:12.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:12.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:56:12.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:56:12.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:56:12.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:11 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:56:12.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:12.148+0000 7fced2a08700 -1 mon.c@2(peon).osd e246 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:12.209 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:56:12.260 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:56:12.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: failed to encode map e244 with expected crc 2024-03-29T04:56:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: Standby manager daemon x restarted 2024-03-29T04:56:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: Standby manager daemon x started 2024-03-29T04:56:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/1598439137' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:56:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/1598439137' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:56:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/1598439137' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:56:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.? 172.21.15.187:0/1598439137' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:56:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: Active manager daemon y restarted 2024-03-29T04:56:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: Activating manager daemon y 2024-03-29T04:56:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: osdmap e245: 8 total, 8 up, 8 in 2024-03-29T04:56:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: mgrmap e33: y(active, starting, since 0.00532876s), standbys: x 2024-03-29T04:56:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: failed to encode map e245 with expected crc 2024-03-29T04:56:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: failed to encode map e245 with expected crc 2024-03-29T04:56:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: failed to encode map e245 with expected crc 2024-03-29T04:56:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:56:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:56:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:56:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi187.vcsnqx"}]: dispatch 2024-03-29T04:56:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi181.pflrwm"}]: dispatch 2024-03-29T04:56:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:56:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:56:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:56:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:56:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:56:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:56:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:56:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:56:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:56:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:56:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:56:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:56:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:56:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: Manager daemon y is now available 2024-03-29T04:56:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:56:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:56:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:56:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:56:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:11 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:56:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:12 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:12.146+0000 7f1d52566700 -1 mon.b@1(peon).osd e246 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:13.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:12 smithi187 ceph-mon[33595]: failed to encode map e245 with expected crc 2024-03-29T04:56:13.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:12 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:12 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:12 smithi187 ceph-mon[33595]: mgrmap e34: y(active, since 1.01241s), standbys: x 2024-03-29T04:56:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:12 smithi187 ceph-mon[33595]: failed to encode map e246 with expected crc 2024-03-29T04:56:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:12 smithi187 ceph-mon[33595]: osdmap e246: 8 total, 8 up, 8 in 2024-03-29T04:56:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:12 smithi187 ceph-mon[33595]: failed to encode map e246 with expected crc 2024-03-29T04:56:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:12 smithi187 ceph-mon[33595]: failed to encode map e246 with expected crc 2024-03-29T04:56:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:12 smithi187 ceph-mon[33595]: failed to encode map e246 with expected crc 2024-03-29T04:56:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:12 smithi187 ceph-mon[33595]: [29/Mar/2024:04:56:12] ENGINE Bus STARTING 2024-03-29T04:56:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:12 smithi187 ceph-mon[33595]: [29/Mar/2024:04:56:12] ENGINE Serving on https://172.21.15.181:7150 2024-03-29T04:56:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:12 smithi187 ceph-mon[33595]: [29/Mar/2024:04:56:12] ENGINE Serving on http://172.21.15.181:8765 2024-03-29T04:56:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:12 smithi187 ceph-mon[33595]: [29/Mar/2024:04:56:12] ENGINE Bus STARTED 2024-03-29T04:56:13.158 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[41183]: failed to encode map e245 with expected crc 2024-03-29T04:56:13.158 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:13.158 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:13.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[41183]: mgrmap e34: y(active, since 1.01241s), standbys: x 2024-03-29T04:56:13.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[41183]: failed to encode map e246 with expected crc 2024-03-29T04:56:13.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[41183]: osdmap e246: 8 total, 8 up, 8 in 2024-03-29T04:56:13.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[41183]: failed to encode map e246 with expected crc 2024-03-29T04:56:13.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[41183]: failed to encode map e246 with expected crc 2024-03-29T04:56:13.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[41183]: failed to encode map e246 with expected crc 2024-03-29T04:56:13.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[41183]: [29/Mar/2024:04:56:12] ENGINE Bus STARTING 2024-03-29T04:56:13.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[41183]: [29/Mar/2024:04:56:12] ENGINE Serving on https://172.21.15.181:7150 2024-03-29T04:56:13.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[41183]: [29/Mar/2024:04:56:12] ENGINE Serving on http://172.21.15.181:8765 2024-03-29T04:56:13.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[41183]: [29/Mar/2024:04:56:12] ENGINE Bus STARTED 2024-03-29T04:56:13.159 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[101970]: failed to encode map e245 with expected crc 2024-03-29T04:56:13.159 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:13.160 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:13.160 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[101970]: mgrmap e34: y(active, since 1.01241s), standbys: x 2024-03-29T04:56:13.160 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[101970]: failed to encode map e246 with expected crc 2024-03-29T04:56:13.160 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[101970]: osdmap e246: 8 total, 8 up, 8 in 2024-03-29T04:56:13.160 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[101970]: failed to encode map e246 with expected crc 2024-03-29T04:56:13.160 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[101970]: failed to encode map e246 with expected crc 2024-03-29T04:56:13.160 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[101970]: failed to encode map e246 with expected crc 2024-03-29T04:56:13.160 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[101970]: [29/Mar/2024:04:56:12] ENGINE Bus STARTING 2024-03-29T04:56:13.160 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[101970]: [29/Mar/2024:04:56:12] ENGINE Serving on https://172.21.15.181:7150 2024-03-29T04:56:13.160 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[101970]: [29/Mar/2024:04:56:12] ENGINE Serving on http://172.21.15.181:8765 2024-03-29T04:56:13.160 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:12 smithi181 ceph-mon[101970]: [29/Mar/2024:04:56:12] ENGINE Bus STARTED 2024-03-29T04:56:13.275 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.TriggerAssertSnapcSeq (5813 ms) 2024-03-29T04:56:13.275 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.SnapTrimBeforePrepare 2024-03-29T04:56:13.305 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:56:13.315 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 41943040~10 2024-03-29T04:56:13.324 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:56:13.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:13.150+0000 7fced2a08700 -1 mon.c@2(peon).osd e247 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:13.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:13 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:13.150+0000 7f1d52566700 -1 mon.b@1(peon).osd e247 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:13.666 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade pause' 2024-03-29T04:56:14.005 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:13 smithi187 ceph-mon[33595]: pgmap v5: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:56:14.005 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:13 smithi187 ceph-mon[33595]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:56:14.005 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:13 smithi187 ceph-mon[33595]: osdmap e247: 8 total, 8 up, 8 in 2024-03-29T04:56:14.006 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:13 smithi187 ceph-mon[33595]: failed to encode map e247 with expected crc 2024-03-29T04:56:14.006 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:13 smithi187 ceph-mon[33595]: failed to encode map e247 with expected crc 2024-03-29T04:56:14.006 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:13 smithi187 ceph-mon[33595]: failed to encode map e247 with expected crc 2024-03-29T04:56:14.006 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:13 smithi187 ceph-mon[33595]: failed to encode map e247 with expected crc 2024-03-29T04:56:14.006 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:13 smithi187 ceph-mon[33595]: failed to encode map e247 with expected crc 2024-03-29T04:56:14.254 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-mon[41183]: pgmap v5: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:56:14.254 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-mon[41183]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:56:14.254 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-mon[41183]: osdmap e247: 8 total, 8 up, 8 in 2024-03-29T04:56:14.254 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-mon[41183]: failed to encode map e247 with expected crc 2024-03-29T04:56:14.254 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-mon[41183]: failed to encode map e247 with expected crc 2024-03-29T04:56:14.254 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-mon[41183]: failed to encode map e247 with expected crc 2024-03-29T04:56:14.254 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-mon[41183]: failed to encode map e247 with expected crc 2024-03-29T04:56:14.255 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-mon[41183]: failed to encode map e247 with expected crc 2024-03-29T04:56:14.255 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:14 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:14.152+0000 7fced2a08700 -1 mon.c@2(peon).osd e248 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:14.255 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-mon[101970]: pgmap v5: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:56:14.255 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-mon[101970]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:56:14.255 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-mon[101970]: osdmap e247: 8 total, 8 up, 8 in 2024-03-29T04:56:14.255 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-mon[101970]: failed to encode map e247 with expected crc 2024-03-29T04:56:14.255 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-mon[101970]: failed to encode map e247 with expected crc 2024-03-29T04:56:14.255 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-mon[101970]: failed to encode map e247 with expected crc 2024-03-29T04:56:14.255 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-mon[101970]: failed to encode map e247 with expected crc 2024-03-29T04:56:14.255 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:13 smithi181 ceph-mon[101970]: failed to encode map e247 with expected crc 2024-03-29T04:56:14.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:14 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:14.152+0000 7f1d52566700 -1 mon.b@1(peon).osd e248 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:14.471 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:15.160 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:14 smithi181 ceph-mon[101970]: mgrmap e35: y(active, since 2s), standbys: x 2024-03-29T04:56:15.160 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:14 smithi181 ceph-mon[101970]: osdmap e248: 8 total, 8 up, 8 in 2024-03-29T04:56:15.160 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:14 smithi181 ceph-mon[101970]: failed to encode map e248 with expected crc 2024-03-29T04:56:15.160 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:14 smithi181 ceph-mon[101970]: failed to encode map e248 with expected crc 2024-03-29T04:56:15.160 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:15 smithi181 ceph-mon[41183]: mgrmap e35: y(active, since 2s), standbys: x 2024-03-29T04:56:15.161 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:15 smithi181 ceph-mon[41183]: osdmap e248: 8 total, 8 up, 8 in 2024-03-29T04:56:15.161 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:15 smithi181 ceph-mon[41183]: failed to encode map e248 with expected crc 2024-03-29T04:56:15.161 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:15 smithi181 ceph-mon[41183]: failed to encode map e248 with expected crc 2024-03-29T04:56:15.173 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 41943040 2024-03-29T04:56:15.190 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:56:15.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:15 smithi187 ceph-mon[33595]: mgrmap e35: y(active, since 2s), standbys: x 2024-03-29T04:56:15.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:15 smithi187 ceph-mon[33595]: osdmap e248: 8 total, 8 up, 8 in 2024-03-29T04:56:15.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:15 smithi187 ceph-mon[33595]: failed to encode map e248 with expected crc 2024-03-29T04:56:15.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:15 smithi187 ceph-mon[33595]: failed to encode map e248 with expected crc 2024-03-29T04:56:15.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:15 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:15.153+0000 7f1d52566700 -1 mon.b@1(peon).osd e249 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:15.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:15.153+0000 7fced2a08700 -1 mon.c@2(peon).osd e249 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:16.302 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:56:16.314 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap3 2024-03-29T04:56:16.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:16 smithi181 ceph-mon[101970]: pgmap v8: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:56:16.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:16 smithi181 ceph-mon[101970]: osdmap e249: 8 total, 8 up, 8 in 2024-03-29T04:56:16.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:16 smithi181 ceph-mon[101970]: failed to encode map e249 with expected crc 2024-03-29T04:56:16.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:16 smithi181 ceph-mon[101970]: failed to encode map e249 with expected crc 2024-03-29T04:56:16.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:16 smithi181 ceph-mon[101970]: mgrmap e36: y(active, since 4s), standbys: x 2024-03-29T04:56:16.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:16 smithi181 ceph-mon[41183]: pgmap v8: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:56:16.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:16 smithi181 ceph-mon[41183]: osdmap e249: 8 total, 8 up, 8 in 2024-03-29T04:56:16.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:16 smithi181 ceph-mon[41183]: failed to encode map e249 with expected crc 2024-03-29T04:56:16.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:16 smithi181 ceph-mon[41183]: failed to encode map e249 with expected crc 2024-03-29T04:56:16.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:16 smithi181 ceph-mon[41183]: mgrmap e36: y(active, since 4s), standbys: x 2024-03-29T04:56:16.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:16 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:16.159+0000 7fced2a08700 -1 mon.c@2(peon).osd e250 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:16.470 INFO:teuthology.orchestra.run.smithi181.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:56:16.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:16 smithi187 ceph-mon[33595]: pgmap v8: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:56:16.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:16 smithi187 ceph-mon[33595]: osdmap e249: 8 total, 8 up, 8 in 2024-03-29T04:56:16.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:16 smithi187 ceph-mon[33595]: failed to encode map e249 with expected crc 2024-03-29T04:56:16.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:16 smithi187 ceph-mon[33595]: failed to encode map e249 with expected crc 2024-03-29T04:56:16.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:16 smithi187 ceph-mon[33595]: mgrmap e36: y(active, since 4s), standbys: x 2024-03-29T04:56:16.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:16.162+0000 7f1d52566700 -1 mon.b@1(peon).osd e250 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:17.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[41183]: osdmap e250: 8 total, 8 up, 8 in 2024-03-29T04:56:17.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[41183]: failed to encode map e250 with expected crc 2024-03-29T04:56:17.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[41183]: failed to encode map e250 with expected crc 2024-03-29T04:56:17.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[41183]: failed to encode map e250 with expected crc 2024-03-29T04:56:17.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[41183]: failed to encode map e250 with expected crc 2024-03-29T04:56:17.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:17.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:17.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[41183]: failed to encode map e250 with expected crc 2024-03-29T04:56:17.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[41183]: from='client.25207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:56:17.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[41183]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:56:17.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:17.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:17.193+0000 7fced2a08700 -1 mon.c@2(peon).osd e251 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:17.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[101970]: osdmap e250: 8 total, 8 up, 8 in 2024-03-29T04:56:17.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[101970]: failed to encode map e250 with expected crc 2024-03-29T04:56:17.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[101970]: failed to encode map e250 with expected crc 2024-03-29T04:56:17.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[101970]: failed to encode map e250 with expected crc 2024-03-29T04:56:17.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[101970]: failed to encode map e250 with expected crc 2024-03-29T04:56:17.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:17.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:17.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[101970]: failed to encode map e250 with expected crc 2024-03-29T04:56:17.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[101970]: from='client.25207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:56:17.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[101970]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:56:17.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:17 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:17.548 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'sleep 60' 2024-03-29T04:56:17.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:17 smithi187 ceph-mon[33595]: osdmap e250: 8 total, 8 up, 8 in 2024-03-29T04:56:17.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:17 smithi187 ceph-mon[33595]: failed to encode map e250 with expected crc 2024-03-29T04:56:17.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:17 smithi187 ceph-mon[33595]: failed to encode map e250 with expected crc 2024-03-29T04:56:17.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:17 smithi187 ceph-mon[33595]: failed to encode map e250 with expected crc 2024-03-29T04:56:17.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:17 smithi187 ceph-mon[33595]: failed to encode map e250 with expected crc 2024-03-29T04:56:17.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:17 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:17.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:17 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:17.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:17 smithi187 ceph-mon[33595]: failed to encode map e250 with expected crc 2024-03-29T04:56:17.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:17 smithi187 ceph-mon[33595]: from='client.25207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:56:17.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:17 smithi187 ceph-mon[33595]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:56:17.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:17 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:17.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:17 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:17.190+0000 7f1d52566700 -1 mon.b@1(peon).osd e251 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:17.953 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:18.219 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 41943040~10 2024-03-29T04:56:18.224 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap4 2024-03-29T04:56:18.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:18 smithi187 ceph-mon[33595]: pgmap v11: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 167 KiB/s rd, 12 KiB/s wr, 196 op/s 2024-03-29T04:56:18.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:18 smithi187 ceph-mon[33595]: failed to encode map e251 with expected crc 2024-03-29T04:56:18.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:18 smithi187 ceph-mon[33595]: osdmap e251: 8 total, 8 up, 8 in 2024-03-29T04:56:18.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:18 smithi187 ceph-mon[33595]: failed to encode map e251 with expected crc 2024-03-29T04:56:18.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:18 smithi187 ceph-mon[33595]: failed to encode map e251 with expected crc 2024-03-29T04:56:18.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:18 smithi187 ceph-mon[33595]: failed to encode map e251 with expected crc 2024-03-29T04:56:18.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:18 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:18.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:18 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:18.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:18.197+0000 7f1d52566700 -1 mon.b@1(peon).osd e252 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:18.408 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-mon[41183]: pgmap v11: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 167 KiB/s rd, 12 KiB/s wr, 196 op/s 2024-03-29T04:56:18.408 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-mon[41183]: failed to encode map e251 with expected crc 2024-03-29T04:56:18.408 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-mon[41183]: osdmap e251: 8 total, 8 up, 8 in 2024-03-29T04:56:18.408 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-mon[41183]: failed to encode map e251 with expected crc 2024-03-29T04:56:18.408 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-mon[41183]: failed to encode map e251 with expected crc 2024-03-29T04:56:18.408 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-mon[41183]: failed to encode map e251 with expected crc 2024-03-29T04:56:18.408 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:18.408 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:18.408 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:18.197+0000 7fced2a08700 -1 mon.c@2(peon).osd e252 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:18.409 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-mon[101970]: pgmap v11: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 167 KiB/s rd, 12 KiB/s wr, 196 op/s 2024-03-29T04:56:18.409 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-mon[101970]: failed to encode map e251 with expected crc 2024-03-29T04:56:18.409 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-mon[101970]: osdmap e251: 8 total, 8 up, 8 in 2024-03-29T04:56:18.409 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-mon[101970]: failed to encode map e251 with expected crc 2024-03-29T04:56:18.409 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-mon[101970]: failed to encode map e251 with expected crc 2024-03-29T04:56:18.409 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-mon[101970]: failed to encode map e251 with expected crc 2024-03-29T04:56:18.409 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:18.409 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:19.134 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:56:18] "GET /metrics HTTP/1.1" 200 35912 "" "Prometheus/2.43.0" 2024-03-29T04:56:19.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:19 smithi181 ceph-mon[101970]: osdmap e252: 8 total, 8 up, 8 in 2024-03-29T04:56:19.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:19 smithi181 ceph-mon[101970]: failed to encode map e252 with expected crc 2024-03-29T04:56:19.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:19 smithi181 ceph-mon[101970]: failed to encode map e252 with expected crc 2024-03-29T04:56:19.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:19 smithi181 ceph-mon[101970]: failed to encode map e252 with expected crc 2024-03-29T04:56:19.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:19 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:19.206+0000 7fced2a08700 -1 mon.c@2(peon).osd e253 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:19.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:19 smithi181 ceph-mon[41183]: osdmap e252: 8 total, 8 up, 8 in 2024-03-29T04:56:19.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:19 smithi181 ceph-mon[41183]: failed to encode map e252 with expected crc 2024-03-29T04:56:19.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:19 smithi181 ceph-mon[41183]: failed to encode map e252 with expected crc 2024-03-29T04:56:19.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:19 smithi181 ceph-mon[41183]: failed to encode map e252 with expected crc 2024-03-29T04:56:19.532 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:19 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:19.203+0000 7f1d52566700 -1 mon.b@1(peon).osd e253 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:19.532 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:19 smithi187 ceph-mon[33595]: osdmap e252: 8 total, 8 up, 8 in 2024-03-29T04:56:19.532 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:19 smithi187 ceph-mon[33595]: failed to encode map e252 with expected crc 2024-03-29T04:56:19.532 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:19 smithi187 ceph-mon[33595]: failed to encode map e252 with expected crc 2024-03-29T04:56:19.533 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:19 smithi187 ceph-mon[33595]: failed to encode map e252 with expected crc 2024-03-29T04:56:20.238 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 41943040 2024-03-29T04:56:20.260 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:56:20.317 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:56:20.327 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:20 smithi181 ceph-mon[41183]: pgmap v14: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 327 KiB/s rd, 24 KiB/s wr, 399 op/s 2024-03-29T04:56:20.327 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:20 smithi181 ceph-mon[41183]: failed to encode map e253 with expected crc 2024-03-29T04:56:20.327 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:20 smithi181 ceph-mon[41183]: osdmap e253: 8 total, 8 up, 8 in 2024-03-29T04:56:20.327 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:20 smithi181 ceph-mon[41183]: failed to encode map e253 with expected crc 2024-03-29T04:56:20.327 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:20 smithi181 ceph-mon[41183]: failed to encode map e253 with expected crc 2024-03-29T04:56:20.327 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:20 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:20.327 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:20 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:20.327 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:20.212+0000 7fced2a08700 -1 mon.c@2(peon).osd e254 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:20.327 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:20 smithi181 ceph-mon[101970]: pgmap v14: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 327 KiB/s rd, 24 KiB/s wr, 399 op/s 2024-03-29T04:56:20.327 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:20 smithi181 ceph-mon[101970]: failed to encode map e253 with expected crc 2024-03-29T04:56:20.328 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:20 smithi181 ceph-mon[101970]: osdmap e253: 8 total, 8 up, 8 in 2024-03-29T04:56:20.328 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:20 smithi181 ceph-mon[101970]: failed to encode map e253 with expected crc 2024-03-29T04:56:20.328 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:20 smithi181 ceph-mon[101970]: failed to encode map e253 with expected crc 2024-03-29T04:56:20.328 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:20 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:20.328 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:20 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:20.604 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:20 smithi187 ceph-mon[33595]: pgmap v14: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 327 KiB/s rd, 24 KiB/s wr, 399 op/s 2024-03-29T04:56:20.604 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:20 smithi187 ceph-mon[33595]: failed to encode map e253 with expected crc 2024-03-29T04:56:20.604 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:20 smithi187 ceph-mon[33595]: osdmap e253: 8 total, 8 up, 8 in 2024-03-29T04:56:20.604 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:20 smithi187 ceph-mon[33595]: failed to encode map e253 with expected crc 2024-03-29T04:56:20.604 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:20 smithi187 ceph-mon[33595]: failed to encode map e253 with expected crc 2024-03-29T04:56:20.605 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:20 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:20.605 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:20 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:20.605 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:20 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:20.212+0000 7f1d52566700 -1 mon.b@1(peon).osd e254 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:20.605 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:20 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 6. 2024-03-29T04:56:20.605 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:20 smithi187 systemd[1]: Stopped Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:56:20.605 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:20 smithi187 systemd[1]: Starting Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:56:21.217 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:21 smithi187 podman[90935]: 2024-03-29 04:56:20.965497573 +0000 UTC m=+0.017926387 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:56:21.217 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:21 smithi187 podman[90935]: 2024-03-29 04:56:21.073804457 +0000 UTC m=+0.126233258 container create e45f2a74fd5b1cce843ba984017a240dd07c20baef84e369b796e32ceb076a6e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, com.redhat.component=ubi8-container, architecture=x86_64, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, io.openshift.expose-services=, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-29T04:56:21.374 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.SnapTrimBeforePrepare (8099 ms) 2024-03-29T04:56:21.374 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.AbortInUseImage 2024-03-29T04:56:21.407 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:56:21.419 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:56:21.512 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:21 smithi187 ceph-mon[33595]: osdmap e254: 8 total, 8 up, 8 in 2024-03-29T04:56:21.512 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:21 smithi187 ceph-mon[33595]: failed to encode map e254 with expected crc 2024-03-29T04:56:21.512 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:21 smithi187 ceph-mon[33595]: failed to encode map e254 with expected crc 2024-03-29T04:56:21.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:21 smithi187 ceph-mon[33595]: failed to encode map e254 with expected crc 2024-03-29T04:56:21.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:21 smithi187 ceph-mon[33595]: failed to encode map e254 with expected crc 2024-03-29T04:56:21.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:21 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:21.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:21 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:21.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:21 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:21.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:21 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:21.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:21 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:56:21.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:21 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:56:21.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:21 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:21.216+0000 7f1d52566700 -1 mon.b@1(peon).osd e255 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:21.513 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:21 smithi187 podman[90935]: 2024-03-29 04:56:21.335651307 +0000 UTC m=+0.388080120 container init e45f2a74fd5b1cce843ba984017a240dd07c20baef84e369b796e32ceb076a6e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, architecture=x86_64, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, name=ubi8, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container) 2024-03-29T04:56:21.513 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:21 smithi187 podman[90935]: 2024-03-29 04:56:21.340763457 +0000 UTC m=+0.393192262 container start e45f2a74fd5b1cce843ba984017a240dd07c20baef84e369b796e32ceb076a6e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, name=ubi8, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vendor=Red Hat, Inc., release=239.1651231664, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, vcs-type=git, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-03-29T04:56:21.513 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:21 smithi187 bash[90935]: e45f2a74fd5b1cce843ba984017a240dd07c20baef84e369b796e32ceb076a6e 2024-03-29T04:56:21.514 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:21 smithi187 systemd[1]: Started Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:56:21.610 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.AbortInUseImage (236 ms) 2024-03-29T04:56:21.610 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.AbortWithoutSnapshots 2024-03-29T04:56:21.641 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:56:21.654 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:56:21.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[41183]: osdmap e254: 8 total, 8 up, 8 in 2024-03-29T04:56:21.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[41183]: failed to encode map e254 with expected crc 2024-03-29T04:56:21.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[41183]: failed to encode map e254 with expected crc 2024-03-29T04:56:21.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[41183]: failed to encode map e254 with expected crc 2024-03-29T04:56:21.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[41183]: failed to encode map e254 with expected crc 2024-03-29T04:56:21.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:21.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:21.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:21.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:21.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:56:21.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:56:21.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:21.218+0000 7fced2a08700 -1 mon.c@2(peon).osd e255 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:21.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[101970]: osdmap e254: 8 total, 8 up, 8 in 2024-03-29T04:56:21.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[101970]: failed to encode map e254 with expected crc 2024-03-29T04:56:21.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[101970]: failed to encode map e254 with expected crc 2024-03-29T04:56:21.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[101970]: failed to encode map e254 with expected crc 2024-03-29T04:56:21.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[101970]: failed to encode map e254 with expected crc 2024-03-29T04:56:21.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:21.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:21.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:21.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:21.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:56:21.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:21 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:56:21.712 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 16777218~8388609 2024-03-29T04:56:21.760 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:56:21.882 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:21 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[91141]: logger=settings t=2024-03-29T04:56:21.506580778Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:56:21.882 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:21 smithi187 podman[91401]: 2024-03-29 04:56:21.532955671 +0000 UTC m=+0.019019139 container died e45f2a74fd5b1cce843ba984017a240dd07c20baef84e369b796e32ceb076a6e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, version=8.5, release=239.1651231664, io.openshift.tags=base rhel8, vcs-type=git, vendor=Red Hat, Inc., io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, distribution-scope=public) 2024-03-29T04:56:21.953 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:56:22.000 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 16777218~8388609 2024-03-29T04:56:22.053 INFO:tasks.workunit.client.0.smithi181.stdout:migration_abort 2024-03-29T04:56:22.259 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.AbortWithoutSnapshots (650 ms) 2024-03-29T04:56:22.260 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.AbortWithSnapshots 2024-03-29T04:56:22.293 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:56:22.304 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:56:22.370 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:56:22.482 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:22 smithi187 podman[91401]: 2024-03-29 04:56:22.218343556 +0000 UTC m=+0.704407012 container remove e45f2a74fd5b1cce843ba984017a240dd07c20baef84e369b796e32ceb076a6e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, maintainer=Paul Cuzner , distribution-scope=public, io.openshift.expose-services=, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, vcs-type=git, version=8.5, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-03-29T04:56:22.482 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:22 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:56:22.482 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: Updating smithi181:/etc/ceph/ceph.conf 2024-03-29T04:56:22.482 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:56:22.482 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: pgmap v17: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 160 KiB/s rd, 12 KiB/s wr, 203 op/s 2024-03-29T04:56:22.482 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: osdmap e255: 8 total, 8 up, 8 in 2024-03-29T04:56:22.483 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: failed to encode map e255 with expected crc 2024-03-29T04:56:22.483 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: failed to encode map e255 with expected crc 2024-03-29T04:56:22.483 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: failed to encode map e255 with expected crc 2024-03-29T04:56:22.483 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: failed to encode map e255 with expected crc 2024-03-29T04:56:22.483 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:56:22.483 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:56:22.483 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:56:22.483 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:56:22.483 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:56:22.483 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:56:22.483 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:22.483 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:22.483 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:22.483 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:22.484 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:22.484 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:22.424+0000 7f1d52566700 -1 mon.b@1(peon).osd e256 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:22.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: Updating smithi181:/etc/ceph/ceph.conf 2024-03-29T04:56:22.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:56:22.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: pgmap v17: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 160 KiB/s rd, 12 KiB/s wr, 203 op/s 2024-03-29T04:56:22.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: osdmap e255: 8 total, 8 up, 8 in 2024-03-29T04:56:22.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: failed to encode map e255 with expected crc 2024-03-29T04:56:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: failed to encode map e255 with expected crc 2024-03-29T04:56:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: failed to encode map e255 with expected crc 2024-03-29T04:56:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: failed to encode map e255 with expected crc 2024-03-29T04:56:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:56:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:56:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:56:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:56:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:56:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:56:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: Updating smithi181:/etc/ceph/ceph.conf 2024-03-29T04:56:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: Updating smithi187:/etc/ceph/ceph.conf 2024-03-29T04:56:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: pgmap v17: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 160 KiB/s rd, 12 KiB/s wr, 203 op/s 2024-03-29T04:56:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: osdmap e255: 8 total, 8 up, 8 in 2024-03-29T04:56:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: failed to encode map e255 with expected crc 2024-03-29T04:56:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: failed to encode map e255 with expected crc 2024-03-29T04:56:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: failed to encode map e255 with expected crc 2024-03-29T04:56:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: failed to encode map e255 with expected crc 2024-03-29T04:56:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:56:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:56:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: Updating smithi187:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:56:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: Updating smithi181:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:56:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: Updating smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:56:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: Updating smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:56:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:22 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:22.426+0000 7fced2a08700 -1 mon.c@2(peon).osd e256 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:22.882 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:22 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:56:23.456 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 8388609~8388609 2024-03-29T04:56:23.469 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 0~1000 2024-03-29T04:56:23.469 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 1010~1000 2024-03-29T04:56:23.469 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap2 2024-03-29T04:56:23.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:23 smithi187 ceph-mon[33595]: pgmap v19: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:56:23.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:23 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:56:23.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:23 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:56:23.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:23 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:56:23.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:23 smithi187 ceph-mon[33595]: pgmap v20: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 MiB/s rd, 27 MiB/s wr, 740 op/s 2024-03-29T04:56:23.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:23 smithi187 ceph-mon[33595]: pgmap v21: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 28 MiB/s wr, 756 op/s 2024-03-29T04:56:23.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:23 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:23.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:23 smithi187 ceph-mon[33595]: osdmap e256: 8 total, 8 up, 8 in 2024-03-29T04:56:23.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:23 smithi187 ceph-mon[33595]: failed to encode map e256 with expected crc 2024-03-29T04:56:23.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:23 smithi187 ceph-mon[33595]: failed to encode map e256 with expected crc 2024-03-29T04:56:23.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:23 smithi187 ceph-mon[33595]: failed to encode map e256 with expected crc 2024-03-29T04:56:23.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:23 smithi187 ceph-mon[33595]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:56:23.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:23 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:23.428+0000 7f1d52566700 -1 mon.b@1(peon).osd e257 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:23.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[101970]: pgmap v19: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:56:23.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:56:23.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:56:23.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:56:23.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[101970]: pgmap v20: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 MiB/s rd, 27 MiB/s wr, 740 op/s 2024-03-29T04:56:23.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[101970]: pgmap v21: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 28 MiB/s wr, 756 op/s 2024-03-29T04:56:23.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:23.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[101970]: osdmap e256: 8 total, 8 up, 8 in 2024-03-29T04:56:23.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[101970]: failed to encode map e256 with expected crc 2024-03-29T04:56:23.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[101970]: failed to encode map e256 with expected crc 2024-03-29T04:56:23.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[101970]: failed to encode map e256 with expected crc 2024-03-29T04:56:23.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[101970]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:56:23.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[41183]: pgmap v19: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:56:23.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:56:23.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:56:23.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:56:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[41183]: pgmap v20: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 MiB/s rd, 27 MiB/s wr, 740 op/s 2024-03-29T04:56:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[41183]: pgmap v21: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 28 MiB/s wr, 756 op/s 2024-03-29T04:56:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:56:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[41183]: osdmap e256: 8 total, 8 up, 8 in 2024-03-29T04:56:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[41183]: failed to encode map e256 with expected crc 2024-03-29T04:56:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[41183]: failed to encode map e256 with expected crc 2024-03-29T04:56:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[41183]: failed to encode map e256 with expected crc 2024-03-29T04:56:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-mon[41183]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:56:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:23 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:23.429+0000 7fced2a08700 -1 mon.c@2(peon).osd e257 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:24.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:24 smithi187 ceph-mon[33595]: failed to encode map e256 with expected crc 2024-03-29T04:56:24.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:24 smithi187 ceph-mon[33595]: osdmap e257: 8 total, 8 up, 8 in 2024-03-29T04:56:24.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:24 smithi187 ceph-mon[33595]: failed to encode map e257 with expected crc 2024-03-29T04:56:24.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:24 smithi187 ceph-mon[33595]: failed to encode map e257 with expected crc 2024-03-29T04:56:24.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:24 smithi187 ceph-mon[33595]: failed to encode map e257 with expected crc 2024-03-29T04:56:24.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:24 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:24.433+0000 7f1d52566700 -1 mon.b@1(peon).osd e258 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:24.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:24 smithi181 ceph-mon[101970]: failed to encode map e256 with expected crc 2024-03-29T04:56:24.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:24 smithi181 ceph-mon[101970]: osdmap e257: 8 total, 8 up, 8 in 2024-03-29T04:56:24.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:24 smithi181 ceph-mon[101970]: failed to encode map e257 with expected crc 2024-03-29T04:56:24.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:24 smithi181 ceph-mon[101970]: failed to encode map e257 with expected crc 2024-03-29T04:56:24.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:24 smithi181 ceph-mon[101970]: failed to encode map e257 with expected crc 2024-03-29T04:56:24.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:24 smithi181 ceph-mon[41183]: failed to encode map e256 with expected crc 2024-03-29T04:56:24.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:24 smithi181 ceph-mon[41183]: osdmap e257: 8 total, 8 up, 8 in 2024-03-29T04:56:24.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:24 smithi181 ceph-mon[41183]: failed to encode map e257 with expected crc 2024-03-29T04:56:24.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:24 smithi181 ceph-mon[41183]: failed to encode map e257 with expected crc 2024-03-29T04:56:24.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:24 smithi181 ceph-mon[41183]: failed to encode map e257 with expected crc 2024-03-29T04:56:24.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:24 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:24.434+0000 7fced2a08700 -1 mon.c@2(peon).osd e258 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:25.467 INFO:tasks.workunit.client.0.smithi181.stdout:write: X 8388609~1000 2024-03-29T04:56:25.473 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 16777228~1000 2024-03-29T04:56:25.473 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 167772160 2024-03-29T04:56:25.483 INFO:tasks.workunit.client.0.smithi181.stdout:write: 2 83886079~8388609 2024-03-29T04:56:25.503 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap3 2024-03-29T04:56:25.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:25 smithi187 ceph-mon[33595]: failed to encode map e257 with expected crc 2024-03-29T04:56:25.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:25 smithi187 ceph-mon[33595]: pgmap v24: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 28 MiB/s wr, 758 op/s 2024-03-29T04:56:25.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:25 smithi187 ceph-mon[33595]: failed to encode map e258 with expected crc 2024-03-29T04:56:25.632 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:25 smithi187 ceph-mon[33595]: failed to encode map e258 with expected crc 2024-03-29T04:56:25.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:25 smithi187 ceph-mon[33595]: osdmap e258: 8 total, 8 up, 8 in 2024-03-29T04:56:25.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:25.436+0000 7f1d52566700 -1 mon.b@1(peon).osd e259 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:25.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:25 smithi181 ceph-mon[101970]: failed to encode map e257 with expected crc 2024-03-29T04:56:25.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:25 smithi181 ceph-mon[101970]: pgmap v24: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 28 MiB/s wr, 758 op/s 2024-03-29T04:56:25.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:25 smithi181 ceph-mon[101970]: failed to encode map e258 with expected crc 2024-03-29T04:56:25.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:25 smithi181 ceph-mon[101970]: failed to encode map e258 with expected crc 2024-03-29T04:56:25.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:25 smithi181 ceph-mon[101970]: osdmap e258: 8 total, 8 up, 8 in 2024-03-29T04:56:25.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:25 smithi181 ceph-mon[41183]: failed to encode map e257 with expected crc 2024-03-29T04:56:25.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:25 smithi181 ceph-mon[41183]: pgmap v24: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 28 MiB/s wr, 758 op/s 2024-03-29T04:56:25.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:25 smithi181 ceph-mon[41183]: failed to encode map e258 with expected crc 2024-03-29T04:56:25.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:25 smithi181 ceph-mon[41183]: failed to encode map e258 with expected crc 2024-03-29T04:56:25.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:25 smithi181 ceph-mon[41183]: osdmap e258: 8 total, 8 up, 8 in 2024-03-29T04:56:25.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:25.438+0000 7fced2a08700 -1 mon.c@2(peon).osd e259 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:26.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:26 smithi181 ceph-mon[101970]: failed to encode map e258 with expected crc 2024-03-29T04:56:26.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:26 smithi181 ceph-mon[101970]: failed to encode map e259 with expected crc 2024-03-29T04:56:26.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:26 smithi181 ceph-mon[101970]: osdmap e259: 8 total, 8 up, 8 in 2024-03-29T04:56:26.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:26 smithi181 ceph-mon[101970]: failed to encode map e259 with expected crc 2024-03-29T04:56:26.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:26 smithi181 ceph-mon[101970]: failed to encode map e259 with expected crc 2024-03-29T04:56:26.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:26 smithi181 ceph-mon[101970]: failed to encode map e259 with expected crc 2024-03-29T04:56:26.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:26 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:26.442+0000 7fced2a08700 -1 mon.c@2(peon).osd e260 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:26 smithi181 ceph-mon[41183]: failed to encode map e258 with expected crc 2024-03-29T04:56:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:26 smithi181 ceph-mon[41183]: failed to encode map e259 with expected crc 2024-03-29T04:56:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:26 smithi181 ceph-mon[41183]: osdmap e259: 8 total, 8 up, 8 in 2024-03-29T04:56:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:26 smithi181 ceph-mon[41183]: failed to encode map e259 with expected crc 2024-03-29T04:56:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:26 smithi181 ceph-mon[41183]: failed to encode map e259 with expected crc 2024-03-29T04:56:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:26 smithi181 ceph-mon[41183]: failed to encode map e259 with expected crc 2024-03-29T04:56:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:26 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:26.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:26 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:26.441+0000 7f1d52566700 -1 mon.b@1(peon).osd e260 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:26.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:26 smithi187 ceph-mon[33595]: failed to encode map e258 with expected crc 2024-03-29T04:56:26.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:26 smithi187 ceph-mon[33595]: failed to encode map e259 with expected crc 2024-03-29T04:56:26.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:26 smithi187 ceph-mon[33595]: osdmap e259: 8 total, 8 up, 8 in 2024-03-29T04:56:26.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:26 smithi187 ceph-mon[33595]: failed to encode map e259 with expected crc 2024-03-29T04:56:26.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:26 smithi187 ceph-mon[33595]: failed to encode map e259 with expected crc 2024-03-29T04:56:26.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:26 smithi187 ceph-mon[33595]: failed to encode map e259 with expected crc 2024-03-29T04:56:26.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:26 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:27.470 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:56:27.495 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 83886079~1 2024-03-29T04:56:27.504 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:56:27.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:27 smithi181 ceph-mon[101970]: pgmap v27: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:56:27.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:27 smithi181 ceph-mon[101970]: osdmap e260: 8 total, 8 up, 8 in 2024-03-29T04:56:27.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:27 smithi181 ceph-mon[101970]: failed to encode map e260 with expected crc 2024-03-29T04:56:27.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:27 smithi181 ceph-mon[101970]: failed to encode map e260 with expected crc 2024-03-29T04:56:27.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:27 smithi181 ceph-mon[41183]: pgmap v27: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:56:27.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:27 smithi181 ceph-mon[41183]: osdmap e260: 8 total, 8 up, 8 in 2024-03-29T04:56:27.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:27 smithi181 ceph-mon[41183]: failed to encode map e260 with expected crc 2024-03-29T04:56:27.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:27 smithi181 ceph-mon[41183]: failed to encode map e260 with expected crc 2024-03-29T04:56:27.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:27 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:27.447+0000 7fced2a08700 -1 mon.c@2(peon).osd e261 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:27.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:27 smithi187 ceph-mon[33595]: pgmap v27: 137 pgs: 137 active+clean; 813 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:56:27.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:27 smithi187 ceph-mon[33595]: osdmap e260: 8 total, 8 up, 8 in 2024-03-29T04:56:27.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:27 smithi187 ceph-mon[33595]: failed to encode map e260 with expected crc 2024-03-29T04:56:27.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:27 smithi187 ceph-mon[33595]: failed to encode map e260 with expected crc 2024-03-29T04:56:27.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:27.449+0000 7f1d52566700 -1 mon.b@1(peon).osd e261 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:28.784 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:28.454+0000 7f1d52566700 -1 mon.b@1(peon).osd e262 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:28.784 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:28 smithi187 ceph-mon[33595]: failed to encode map e260 with expected crc 2024-03-29T04:56:28.784 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:28 smithi187 ceph-mon[33595]: osdmap e261: 8 total, 8 up, 8 in 2024-03-29T04:56:28.784 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:28 smithi187 ceph-mon[33595]: failed to encode map e261 with expected crc 2024-03-29T04:56:28.784 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:28 smithi187 ceph-mon[33595]: failed to encode map e261 with expected crc 2024-03-29T04:56:28.784 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:28 smithi187 ceph-mon[33595]: failed to encode map e261 with expected crc 2024-03-29T04:56:28.785 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:56:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:56:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:56:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:28 smithi181 ceph-mon[101970]: failed to encode map e260 with expected crc 2024-03-29T04:56:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:28 smithi181 ceph-mon[101970]: osdmap e261: 8 total, 8 up, 8 in 2024-03-29T04:56:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:28 smithi181 ceph-mon[101970]: failed to encode map e261 with expected crc 2024-03-29T04:56:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:28 smithi181 ceph-mon[101970]: failed to encode map e261 with expected crc 2024-03-29T04:56:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:28 smithi181 ceph-mon[101970]: failed to encode map e261 with expected crc 2024-03-29T04:56:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:28.455+0000 7fced2a08700 -1 mon.c@2(peon).osd e262 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:28 smithi181 ceph-mon[41183]: failed to encode map e260 with expected crc 2024-03-29T04:56:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:28 smithi181 ceph-mon[41183]: osdmap e261: 8 total, 8 up, 8 in 2024-03-29T04:56:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:28 smithi181 ceph-mon[41183]: failed to encode map e261 with expected crc 2024-03-29T04:56:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:28 smithi181 ceph-mon[41183]: failed to encode map e261 with expected crc 2024-03-29T04:56:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:28 smithi181 ceph-mon[41183]: failed to encode map e261 with expected crc 2024-03-29T04:56:29.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:56:28] "GET /metrics HTTP/1.1" 200 38725 "" "Prometheus/2.43.0" 2024-03-29T04:56:29.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:29 smithi187 ceph-mon[33595]: pgmap v30: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 8.1 MiB/s rd, 12 MiB/s wr, 169 op/s 2024-03-29T04:56:29.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:29 smithi187 ceph-mon[33595]: osdmap e262: 8 total, 8 up, 8 in 2024-03-29T04:56:29.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:29 smithi187 ceph-mon[33595]: failed to encode map e262 with expected crc 2024-03-29T04:56:29.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:29 smithi187 ceph-mon[33595]: failed to encode map e262 with expected crc 2024-03-29T04:56:29.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:29 smithi187 ceph-mon[33595]: failed to encode map e262 with expected crc 2024-03-29T04:56:29.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:29 smithi187 ceph-mon[33595]: failed to encode map e262 with expected crc 2024-03-29T04:56:29.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:29 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:29.460+0000 7f1d52566700 -1 mon.b@1(peon).osd e263 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:29.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:29 smithi181 ceph-mon[101970]: pgmap v30: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 8.1 MiB/s rd, 12 MiB/s wr, 169 op/s 2024-03-29T04:56:29.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:29 smithi181 ceph-mon[101970]: osdmap e262: 8 total, 8 up, 8 in 2024-03-29T04:56:29.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:29 smithi181 ceph-mon[101970]: failed to encode map e262 with expected crc 2024-03-29T04:56:29.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:29 smithi181 ceph-mon[101970]: failed to encode map e262 with expected crc 2024-03-29T04:56:29.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:29 smithi181 ceph-mon[101970]: failed to encode map e262 with expected crc 2024-03-29T04:56:29.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:29 smithi181 ceph-mon[101970]: failed to encode map e262 with expected crc 2024-03-29T04:56:29.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:29 smithi181 ceph-mon[41183]: pgmap v30: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 8.1 MiB/s rd, 12 MiB/s wr, 169 op/s 2024-03-29T04:56:29.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:29 smithi181 ceph-mon[41183]: osdmap e262: 8 total, 8 up, 8 in 2024-03-29T04:56:29.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:29 smithi181 ceph-mon[41183]: failed to encode map e262 with expected crc 2024-03-29T04:56:29.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:29 smithi181 ceph-mon[41183]: failed to encode map e262 with expected crc 2024-03-29T04:56:29.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:29 smithi181 ceph-mon[41183]: failed to encode map e262 with expected crc 2024-03-29T04:56:29.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:29 smithi181 ceph-mon[41183]: failed to encode map e262 with expected crc 2024-03-29T04:56:29.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:29 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:29.460+0000 7fced2a08700 -1 mon.c@2(peon).osd e263 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:30.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:30 smithi187 ceph-mon[33595]: osdmap e263: 8 total, 8 up, 8 in 2024-03-29T04:56:30.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:30 smithi187 ceph-mon[33595]: failed to encode map e263 with expected crc 2024-03-29T04:56:30.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:30 smithi187 ceph-mon[33595]: failed to encode map e263 with expected crc 2024-03-29T04:56:30.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:30 smithi187 ceph-mon[33595]: failed to encode map e263 with expected crc 2024-03-29T04:56:30.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:30 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:30.466+0000 7f1d52566700 -1 mon.b@1(peon).osd e264 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:30.943 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:56:30.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:30 smithi181 ceph-mon[101970]: osdmap e263: 8 total, 8 up, 8 in 2024-03-29T04:56:30.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:30 smithi181 ceph-mon[101970]: failed to encode map e263 with expected crc 2024-03-29T04:56:30.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:30 smithi181 ceph-mon[101970]: failed to encode map e263 with expected crc 2024-03-29T04:56:30.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:30 smithi181 ceph-mon[101970]: failed to encode map e263 with expected crc 2024-03-29T04:56:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:30 smithi181 ceph-mon[41183]: osdmap e263: 8 total, 8 up, 8 in 2024-03-29T04:56:30.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:30 smithi181 ceph-mon[41183]: failed to encode map e263 with expected crc 2024-03-29T04:56:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:30 smithi181 ceph-mon[41183]: failed to encode map e263 with expected crc 2024-03-29T04:56:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:30 smithi181 ceph-mon[41183]: failed to encode map e263 with expected crc 2024-03-29T04:56:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:30.466+0000 7fced2a08700 -1 mon.c@2(peon).osd e264 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:31.041 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 16777218~8388609 2024-03-29T04:56:31.498 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 0~8388609 2024-03-29T04:56:31.514 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 16777218~8388609 2024-03-29T04:56:31.592 INFO:tasks.workunit.client.0.smithi181.stdout:migration_abort 2024-03-29T04:56:31.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:31 smithi187 ceph-mon[33595]: pgmap v33: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 8.1 MiB/s rd, 12 MiB/s wr, 169 op/s 2024-03-29T04:56:31.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:31 smithi187 ceph-mon[33595]: osdmap e264: 8 total, 8 up, 8 in 2024-03-29T04:56:31.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:31 smithi187 ceph-mon[33595]: failed to encode map e264 with expected crc 2024-03-29T04:56:31.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:31 smithi187 ceph-mon[33595]: failed to encode map e264 with expected crc 2024-03-29T04:56:31.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:31 smithi187 ceph-mon[33595]: failed to encode map e264 with expected crc 2024-03-29T04:56:31.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:31 smithi187 ceph-mon[33595]: failed to encode map e264 with expected crc 2024-03-29T04:56:31.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:31 smithi187 ceph-mon[33595]: failed to encode map e264 with expected crc 2024-03-29T04:56:31.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:31.473+0000 7f1d52566700 -1 mon.b@1(peon).osd e265 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:31.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:31 smithi181 ceph-mon[101970]: pgmap v33: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 8.1 MiB/s rd, 12 MiB/s wr, 169 op/s 2024-03-29T04:56:31.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:31 smithi181 ceph-mon[101970]: osdmap e264: 8 total, 8 up, 8 in 2024-03-29T04:56:31.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:31 smithi181 ceph-mon[101970]: failed to encode map e264 with expected crc 2024-03-29T04:56:31.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:31 smithi181 ceph-mon[101970]: failed to encode map e264 with expected crc 2024-03-29T04:56:31.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:31 smithi181 ceph-mon[101970]: failed to encode map e264 with expected crc 2024-03-29T04:56:31.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:31 smithi181 ceph-mon[101970]: failed to encode map e264 with expected crc 2024-03-29T04:56:31.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:31 smithi181 ceph-mon[101970]: failed to encode map e264 with expected crc 2024-03-29T04:56:31.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:31 smithi181 ceph-mon[41183]: pgmap v33: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 8.1 MiB/s rd, 12 MiB/s wr, 169 op/s 2024-03-29T04:56:31.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:31 smithi181 ceph-mon[41183]: osdmap e264: 8 total, 8 up, 8 in 2024-03-29T04:56:31.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:31 smithi181 ceph-mon[41183]: failed to encode map e264 with expected crc 2024-03-29T04:56:31.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:31 smithi181 ceph-mon[41183]: failed to encode map e264 with expected crc 2024-03-29T04:56:31.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:31 smithi181 ceph-mon[41183]: failed to encode map e264 with expected crc 2024-03-29T04:56:31.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:31 smithi181 ceph-mon[41183]: failed to encode map e264 with expected crc 2024-03-29T04:56:31.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:31 smithi181 ceph-mon[41183]: failed to encode map e264 with expected crc 2024-03-29T04:56:31.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:31.473+0000 7fced2a08700 -1 mon.c@2(peon).osd e265 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:32.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:32 smithi181 ceph-mon[101970]: osdmap e265: 8 total, 8 up, 8 in 2024-03-29T04:56:32.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:32 smithi181 ceph-mon[101970]: failed to encode map e265 with expected crc 2024-03-29T04:56:32.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:32 smithi181 ceph-mon[101970]: failed to encode map e265 with expected crc 2024-03-29T04:56:32.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:32 smithi181 ceph-mon[101970]: failed to encode map e265 with expected crc 2024-03-29T04:56:32.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:32 smithi181 ceph-mon[101970]: failed to encode map e265 with expected crc 2024-03-29T04:56:32.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:32 smithi181 ceph-mon[101970]: osdmap e266: 8 total, 8 up, 8 in 2024-03-29T04:56:32.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:32 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:32.428+0000 7fced2a08700 -1 mon.c@2(peon).osd e266 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:32.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:32 smithi181 ceph-mon[41183]: osdmap e265: 8 total, 8 up, 8 in 2024-03-29T04:56:32.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:32 smithi181 ceph-mon[41183]: failed to encode map e265 with expected crc 2024-03-29T04:56:32.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:32 smithi181 ceph-mon[41183]: failed to encode map e265 with expected crc 2024-03-29T04:56:32.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:32 smithi181 ceph-mon[41183]: failed to encode map e265 with expected crc 2024-03-29T04:56:32.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:32 smithi181 ceph-mon[41183]: failed to encode map e265 with expected crc 2024-03-29T04:56:32.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:32 smithi181 ceph-mon[41183]: osdmap e266: 8 total, 8 up, 8 in 2024-03-29T04:56:32.863 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:32 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 7. 2024-03-29T04:56:32.863 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:32 smithi187 systemd[1]: Stopped Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:56:32.863 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:32 smithi187 systemd[1]: Starting Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:56:32.863 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:32 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:32.428+0000 7f1d52566700 -1 mon.b@1(peon).osd e266 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:32.863 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:32 smithi187 ceph-mon[33595]: osdmap e265: 8 total, 8 up, 8 in 2024-03-29T04:56:32.863 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:32 smithi187 ceph-mon[33595]: failed to encode map e265 with expected crc 2024-03-29T04:56:32.864 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:32 smithi187 ceph-mon[33595]: failed to encode map e265 with expected crc 2024-03-29T04:56:32.864 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:32 smithi187 ceph-mon[33595]: failed to encode map e265 with expected crc 2024-03-29T04:56:32.864 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:32 smithi187 ceph-mon[33595]: failed to encode map e265 with expected crc 2024-03-29T04:56:32.864 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:32 smithi187 ceph-mon[33595]: osdmap e266: 8 total, 8 up, 8 in 2024-03-29T04:56:33.127 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:32 smithi187 podman[91992]: 2024-03-29 04:56:32.759745528 +0000 UTC m=+0.017344063 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:56:33.127 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:32 smithi187 podman[91992]: 2024-03-29 04:56:32.877348371 +0000 UTC m=+0.134946891 container create f90dc97b94bf1a75a2fbc79aadd426b6438330b6f58ec54529243ab93f0073cb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, version=8.5, io.openshift.tags=base rhel8, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , release=239.1651231664, vcs-type=git) 2024-03-29T04:56:33.382 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:33 smithi187 podman[91992]: 2024-03-29 04:56:33.121277081 +0000 UTC m=+0.378875608 container init f90dc97b94bf1a75a2fbc79aadd426b6438330b6f58ec54529243ab93f0073cb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, description=Ceph Grafana Container, maintainer=Paul Cuzner , name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., release=239.1651231664, vcs-type=git) 2024-03-29T04:56:33.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:33 smithi187 podman[91992]: 2024-03-29 04:56:33.127145157 +0000 UTC m=+0.384743687 container start f90dc97b94bf1a75a2fbc79aadd426b6438330b6f58ec54529243ab93f0073cb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, description=Ceph Grafana Container, distribution-scope=public, name=ubi8, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, architecture=x86_64, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., io.buildah.version=1.32.0, vcs-type=git) 2024-03-29T04:56:33.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:33 smithi187 bash[91992]: f90dc97b94bf1a75a2fbc79aadd426b6438330b6f58ec54529243ab93f0073cb 2024-03-29T04:56:33.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:33 smithi187 systemd[1]: Started Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:56:33.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:33 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[92006]: logger=settings t=2024-03-29T04:56:33.276404234Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:56:33.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:33 smithi187 podman[92042]: 2024-03-29 04:56:33.306101246 +0000 UTC m=+0.018242292 container died f90dc97b94bf1a75a2fbc79aadd426b6438330b6f58ec54529243ab93f0073cb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, name=ubi8, version=8.5, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, vcs-type=git, io.buildah.version=1.32.0, distribution-scope=public, io.openshift.expose-services=, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-03-29T04:56:33.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:33 smithi181 ceph-mon[101970]: pgmap v36: 137 pgs: 137 active+clean; 945 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 464 op/s 2024-03-29T04:56:33.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:33 smithi181 ceph-mon[101970]: failed to encode map e266 with expected crc 2024-03-29T04:56:33.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:33 smithi181 ceph-mon[101970]: failed to encode map e266 with expected crc 2024-03-29T04:56:33.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:33 smithi181 ceph-mon[101970]: failed to encode map e266 with expected crc 2024-03-29T04:56:33.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:33 smithi181 ceph-mon[101970]: osdmap e267: 8 total, 8 up, 8 in 2024-03-29T04:56:33.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:33 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:33.432+0000 7fced2a08700 -1 mon.c@2(peon).osd e267 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:33.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:33 smithi181 ceph-mon[41183]: pgmap v36: 137 pgs: 137 active+clean; 945 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 464 op/s 2024-03-29T04:56:33.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:33 smithi181 ceph-mon[41183]: failed to encode map e266 with expected crc 2024-03-29T04:56:33.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:33 smithi181 ceph-mon[41183]: failed to encode map e266 with expected crc 2024-03-29T04:56:33.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:33 smithi181 ceph-mon[41183]: failed to encode map e266 with expected crc 2024-03-29T04:56:33.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:33 smithi181 ceph-mon[41183]: osdmap e267: 8 total, 8 up, 8 in 2024-03-29T04:56:33.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:33 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:33.432+0000 7f1d52566700 -1 mon.b@1(peon).osd e267 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:33.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:33 smithi187 ceph-mon[33595]: pgmap v36: 137 pgs: 137 active+clean; 945 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 464 op/s 2024-03-29T04:56:33.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:33 smithi187 ceph-mon[33595]: failed to encode map e266 with expected crc 2024-03-29T04:56:33.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:33 smithi187 ceph-mon[33595]: failed to encode map e266 with expected crc 2024-03-29T04:56:33.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:33 smithi187 ceph-mon[33595]: failed to encode map e266 with expected crc 2024-03-29T04:56:33.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:33 smithi187 ceph-mon[33595]: osdmap e267: 8 total, 8 up, 8 in 2024-03-29T04:56:34.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:33 smithi187 podman[92042]: 2024-03-29 04:56:33.933508679 +0000 UTC m=+0.645649759 container remove f90dc97b94bf1a75a2fbc79aadd426b6438330b6f58ec54529243ab93f0073cb (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, distribution-scope=public, description=Ceph Grafana Container, io.openshift.expose-services=, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-03-29T04:56:34.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:33 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:56:34.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:34 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:56:34.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:34 smithi181 ceph-mon[101970]: failed to encode map e266 with expected crc 2024-03-29T04:56:34.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:34 smithi181 ceph-mon[101970]: failed to encode map e267 with expected crc 2024-03-29T04:56:34.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:34 smithi181 ceph-mon[101970]: failed to encode map e267 with expected crc 2024-03-29T04:56:34.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:34 smithi181 ceph-mon[101970]: failed to encode map e267 with expected crc 2024-03-29T04:56:34.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:34 smithi181 ceph-mon[101970]: failed to encode map e267 with expected crc 2024-03-29T04:56:34.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:34 smithi181 ceph-mon[101970]: osdmap e268: 8 total, 8 up, 8 in 2024-03-29T04:56:34.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:34 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:34.433+0000 7fced2a08700 -1 mon.c@2(peon).osd e268 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:34.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:34 smithi181 ceph-mon[41183]: failed to encode map e266 with expected crc 2024-03-29T04:56:34.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:34 smithi181 ceph-mon[41183]: failed to encode map e267 with expected crc 2024-03-29T04:56:34.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:34 smithi181 ceph-mon[41183]: failed to encode map e267 with expected crc 2024-03-29T04:56:34.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:34 smithi181 ceph-mon[41183]: failed to encode map e267 with expected crc 2024-03-29T04:56:34.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:34 smithi181 ceph-mon[41183]: failed to encode map e267 with expected crc 2024-03-29T04:56:34.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:34 smithi181 ceph-mon[41183]: osdmap e268: 8 total, 8 up, 8 in 2024-03-29T04:56:34.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:34 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:34.434+0000 7f1d52566700 -1 mon.b@1(peon).osd e268 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:34.945 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:34 smithi187 ceph-mon[33595]: failed to encode map e266 with expected crc 2024-03-29T04:56:34.945 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:34 smithi187 ceph-mon[33595]: failed to encode map e267 with expected crc 2024-03-29T04:56:34.945 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:34 smithi187 ceph-mon[33595]: failed to encode map e267 with expected crc 2024-03-29T04:56:34.945 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:34 smithi187 ceph-mon[33595]: failed to encode map e267 with expected crc 2024-03-29T04:56:34.945 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:34 smithi187 ceph-mon[33595]: failed to encode map e267 with expected crc 2024-03-29T04:56:34.945 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:34 smithi187 ceph-mon[33595]: osdmap e268: 8 total, 8 up, 8 in 2024-03-29T04:56:35.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:35 smithi181 ceph-mon[101970]: pgmap v39: 137 pgs: 137 active+clean; 945 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 464 op/s 2024-03-29T04:56:35.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:35 smithi181 ceph-mon[101970]: failed to encode map e268 with expected crc 2024-03-29T04:56:35.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:35 smithi181 ceph-mon[101970]: failed to encode map e268 with expected crc 2024-03-29T04:56:35.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:35 smithi181 ceph-mon[101970]: failed to encode map e268 with expected crc 2024-03-29T04:56:35.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:35 smithi181 ceph-mon[101970]: failed to encode map e268 with expected crc 2024-03-29T04:56:35.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:35 smithi181 ceph-mon[101970]: osdmap e269: 8 total, 8 up, 8 in 2024-03-29T04:56:35.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:35 smithi181 ceph-mon[101970]: failed to encode map e269 with expected crc 2024-03-29T04:56:35.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:35 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:35.435+0000 7fced2a08700 -1 mon.c@2(peon).osd e269 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:35.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:35 smithi181 ceph-mon[41183]: pgmap v39: 137 pgs: 137 active+clean; 945 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 464 op/s 2024-03-29T04:56:35.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:35 smithi181 ceph-mon[41183]: failed to encode map e268 with expected crc 2024-03-29T04:56:35.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:35 smithi181 ceph-mon[41183]: failed to encode map e268 with expected crc 2024-03-29T04:56:35.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:35 smithi181 ceph-mon[41183]: failed to encode map e268 with expected crc 2024-03-29T04:56:35.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:35 smithi181 ceph-mon[41183]: failed to encode map e268 with expected crc 2024-03-29T04:56:35.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:35 smithi181 ceph-mon[41183]: osdmap e269: 8 total, 8 up, 8 in 2024-03-29T04:56:35.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:35 smithi181 ceph-mon[41183]: failed to encode map e269 with expected crc 2024-03-29T04:56:35.784 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.AbortWithSnapshots (13525 ms) 2024-03-29T04:56:35.784 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.CloneV1Parent 2024-03-29T04:56:35.813 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:56:35.823 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 0~10 2024-03-29T04:56:35.832 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:56:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:35 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:35.435+0000 7f1d52566700 -1 mon.b@1(peon).osd e269 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:35 smithi187 ceph-mon[33595]: pgmap v39: 137 pgs: 137 active+clean; 945 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 464 op/s 2024-03-29T04:56:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:35 smithi187 ceph-mon[33595]: failed to encode map e268 with expected crc 2024-03-29T04:56:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:35 smithi187 ceph-mon[33595]: failed to encode map e268 with expected crc 2024-03-29T04:56:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:35 smithi187 ceph-mon[33595]: failed to encode map e268 with expected crc 2024-03-29T04:56:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:35 smithi187 ceph-mon[33595]: failed to encode map e268 with expected crc 2024-03-29T04:56:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:35 smithi187 ceph-mon[33595]: osdmap e269: 8 total, 8 up, 8 in 2024-03-29T04:56:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:35 smithi187 ceph-mon[33595]: failed to encode map e269 with expected crc 2024-03-29T04:56:36.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:36 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:36.494+0000 7f1d52566700 -1 mon.b@1(peon).osd e270 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:36.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:36 smithi187 ceph-mon[33595]: failed to encode map e269 with expected crc 2024-03-29T04:56:36.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:36 smithi187 ceph-mon[33595]: failed to encode map e269 with expected crc 2024-03-29T04:56:36.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:36 smithi181 ceph-mon[101970]: failed to encode map e269 with expected crc 2024-03-29T04:56:36.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:36 smithi181 ceph-mon[101970]: failed to encode map e269 with expected crc 2024-03-29T04:56:36.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:36.494+0000 7fced2a08700 -1 mon.c@2(peon).osd e270 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:36.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:36 smithi181 ceph-mon[41183]: failed to encode map e269 with expected crc 2024-03-29T04:56:36.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:36 smithi181 ceph-mon[41183]: failed to encode map e269 with expected crc 2024-03-29T04:56:37.528 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap1 2024-03-29T04:56:37.588 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:56:37.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:37 smithi187 ceph-mon[33595]: pgmap v42: 137 pgs: 137 active+clean; 945 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:56:37.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:37 smithi187 ceph-mon[33595]: osdmap e270: 8 total, 8 up, 8 in 2024-03-29T04:56:37.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:37 smithi187 ceph-mon[33595]: failed to encode map e270 with expected crc 2024-03-29T04:56:37.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:37 smithi187 ceph-mon[33595]: failed to encode map e270 with expected crc 2024-03-29T04:56:37.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:37 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:37.500+0000 7f1d52566700 -1 mon.b@1(peon).osd e271 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:37.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:37 smithi181 ceph-mon[41183]: pgmap v42: 137 pgs: 137 active+clean; 945 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:56:37.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:37 smithi181 ceph-mon[41183]: osdmap e270: 8 total, 8 up, 8 in 2024-03-29T04:56:37.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:37 smithi181 ceph-mon[41183]: failed to encode map e270 with expected crc 2024-03-29T04:56:37.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:37 smithi181 ceph-mon[41183]: failed to encode map e270 with expected crc 2024-03-29T04:56:37.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:37 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:37.501+0000 7fced2a08700 -1 mon.c@2(peon).osd e271 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:37.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:37 smithi181 ceph-mon[101970]: pgmap v42: 137 pgs: 137 active+clean; 945 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:56:37.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:37 smithi181 ceph-mon[101970]: osdmap e270: 8 total, 8 up, 8 in 2024-03-29T04:56:37.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:37 smithi181 ceph-mon[101970]: failed to encode map e270 with expected crc 2024-03-29T04:56:37.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:37 smithi181 ceph-mon[101970]: failed to encode map e270 with expected crc 2024-03-29T04:56:38.739 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:56:38.813 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:56:38.853 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:38 smithi181 ceph-mon[101970]: osdmap e271: 8 total, 8 up, 8 in 2024-03-29T04:56:38.853 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:38 smithi181 ceph-mon[101970]: failed to encode map e271 with expected crc 2024-03-29T04:56:38.853 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:38 smithi181 ceph-mon[101970]: failed to encode map e271 with expected crc 2024-03-29T04:56:38.853 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:38 smithi181 ceph-mon[101970]: failed to encode map e271 with expected crc 2024-03-29T04:56:38.853 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:38 smithi181 ceph-mon[101970]: failed to encode map e271 with expected crc 2024-03-29T04:56:38.853 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:38.510+0000 7fced2a08700 -1 mon.c@2(peon).osd e272 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:38.853 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:38 smithi181 ceph-mon[41183]: osdmap e271: 8 total, 8 up, 8 in 2024-03-29T04:56:38.853 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:38 smithi181 ceph-mon[41183]: failed to encode map e271 with expected crc 2024-03-29T04:56:38.853 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:38 smithi181 ceph-mon[41183]: failed to encode map e271 with expected crc 2024-03-29T04:56:38.853 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:38 smithi181 ceph-mon[41183]: failed to encode map e271 with expected crc 2024-03-29T04:56:38.854 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:38 smithi181 ceph-mon[41183]: failed to encode map e271 with expected crc 2024-03-29T04:56:38.882 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:56:38 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:56:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:56:38.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:38 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:38.508+0000 7f1d52566700 -1 mon.b@1(peon).osd e272 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:38.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:38 smithi187 ceph-mon[33595]: osdmap e271: 8 total, 8 up, 8 in 2024-03-29T04:56:38.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:38 smithi187 ceph-mon[33595]: failed to encode map e271 with expected crc 2024-03-29T04:56:38.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:38 smithi187 ceph-mon[33595]: failed to encode map e271 with expected crc 2024-03-29T04:56:38.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:38 smithi187 ceph-mon[33595]: failed to encode map e271 with expected crc 2024-03-29T04:56:38.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:38 smithi187 ceph-mon[33595]: failed to encode map e271 with expected crc 2024-03-29T04:56:39.126 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:56:38] "GET /metrics HTTP/1.1" 200 38725 "" "Prometheus/2.43.0" 2024-03-29T04:56:39.628 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.CloneV1Parent (3844 ms) 2024-03-29T04:56:39.629 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.CloneV2Parent 2024-03-29T04:56:39.658 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:56:39.668 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 0~10 2024-03-29T04:56:39.675 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:56:39.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:39 smithi187 ceph-mon[33595]: pgmap v45: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 11 KiB/s wr, 129 op/s 2024-03-29T04:56:39.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:39 smithi187 ceph-mon[33595]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:39 smithi187 ceph-mon[33595]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:39 smithi187 ceph-mon[33595]: osdmap e272: 8 total, 8 up, 8 in 2024-03-29T04:56:39.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:39 smithi187 ceph-mon[33595]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:39 smithi187 ceph-mon[33595]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:39 smithi187 ceph-mon[33595]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:39 smithi187 ceph-mon[33595]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:39 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:39.513+0000 7f1d52566700 -1 mon.b@1(peon).osd e273 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:39.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-mon[41183]: pgmap v45: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 11 KiB/s wr, 129 op/s 2024-03-29T04:56:39.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-mon[41183]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-mon[41183]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-mon[41183]: osdmap e272: 8 total, 8 up, 8 in 2024-03-29T04:56:39.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-mon[41183]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-mon[41183]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-mon[41183]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-mon[41183]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:39.513+0000 7fced2a08700 -1 mon.c@2(peon).osd e273 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:39.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-mon[101970]: pgmap v45: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 11 KiB/s wr, 129 op/s 2024-03-29T04:56:39.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-mon[101970]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-mon[101970]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-mon[101970]: osdmap e272: 8 total, 8 up, 8 in 2024-03-29T04:56:39.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-mon[101970]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-mon[101970]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-mon[101970]: failed to encode map e272 with expected crc 2024-03-29T04:56:39.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:39 smithi181 ceph-mon[101970]: failed to encode map e272 with expected crc 2024-03-29T04:56:40.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:40.518+0000 7f1d52566700 -1 mon.b@1(peon).osd e274 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:40.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:40 smithi187 ceph-mon[33595]: osdmap e273: 8 total, 8 up, 8 in 2024-03-29T04:56:40.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:40 smithi187 ceph-mon[33595]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:40 smithi187 ceph-mon[33595]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:40 smithi187 ceph-mon[33595]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:40 smithi187 ceph-mon[33595]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:40 smithi187 ceph-mon[33595]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:40 smithi187 ceph-mon[33595]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:40 smithi187 ceph-mon[33595]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:40.518+0000 7fced2a08700 -1 mon.c@2(peon).osd e274 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:40.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-mon[41183]: osdmap e273: 8 total, 8 up, 8 in 2024-03-29T04:56:40.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-mon[41183]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-mon[41183]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-mon[41183]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-mon[41183]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-mon[41183]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-mon[41183]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-mon[41183]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-mon[101970]: osdmap e273: 8 total, 8 up, 8 in 2024-03-29T04:56:40.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-mon[101970]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-mon[101970]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-mon[101970]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-mon[101970]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-mon[101970]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-mon[101970]: failed to encode map e273 with expected crc 2024-03-29T04:56:40.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:40 smithi181 ceph-mon[101970]: failed to encode map e273 with expected crc 2024-03-29T04:56:41.553 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap1 2024-03-29T04:56:41.616 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:56:41.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:41 smithi187 ceph-mon[33595]: pgmap v48: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 11 KiB/s wr, 129 op/s 2024-03-29T04:56:41.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:41 smithi187 ceph-mon[33595]: osdmap e274: 8 total, 8 up, 8 in 2024-03-29T04:56:41.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:41 smithi187 ceph-mon[33595]: failed to encode map e274 with expected crc 2024-03-29T04:56:41.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:41 smithi187 ceph-mon[33595]: failed to encode map e274 with expected crc 2024-03-29T04:56:41.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:41 smithi187 ceph-mon[33595]: failed to encode map e274 with expected crc 2024-03-29T04:56:41.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:41 smithi187 ceph-mon[33595]: failed to encode map e274 with expected crc 2024-03-29T04:56:41.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:41 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:41.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:41 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:41.525+0000 7f1d52566700 -1 mon.b@1(peon).osd e275 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:41.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:41 smithi181 ceph-mon[101970]: pgmap v48: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 11 KiB/s wr, 129 op/s 2024-03-29T04:56:41.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:41 smithi181 ceph-mon[101970]: osdmap e274: 8 total, 8 up, 8 in 2024-03-29T04:56:41.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:41 smithi181 ceph-mon[101970]: failed to encode map e274 with expected crc 2024-03-29T04:56:41.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:41 smithi181 ceph-mon[101970]: failed to encode map e274 with expected crc 2024-03-29T04:56:41.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:41 smithi181 ceph-mon[101970]: failed to encode map e274 with expected crc 2024-03-29T04:56:41.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:41 smithi181 ceph-mon[101970]: failed to encode map e274 with expected crc 2024-03-29T04:56:41.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:41.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:41 smithi181 ceph-mon[41183]: pgmap v48: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 11 KiB/s wr, 129 op/s 2024-03-29T04:56:41.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:41 smithi181 ceph-mon[41183]: osdmap e274: 8 total, 8 up, 8 in 2024-03-29T04:56:41.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:41 smithi181 ceph-mon[41183]: failed to encode map e274 with expected crc 2024-03-29T04:56:41.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:41 smithi181 ceph-mon[41183]: failed to encode map e274 with expected crc 2024-03-29T04:56:41.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:41 smithi181 ceph-mon[41183]: failed to encode map e274 with expected crc 2024-03-29T04:56:41.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:41 smithi181 ceph-mon[41183]: failed to encode map e274 with expected crc 2024-03-29T04:56:41.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:41 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:41.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:41 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:41.524+0000 7fced2a08700 -1 mon.c@2(peon).osd e275 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:42.663 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:56:42.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:42 smithi181 ceph-mon[101970]: osdmap e275: 8 total, 8 up, 8 in 2024-03-29T04:56:42.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:42 smithi181 ceph-mon[101970]: failed to encode map e275 with expected crc 2024-03-29T04:56:42.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:42 smithi181 ceph-mon[101970]: failed to encode map e275 with expected crc 2024-03-29T04:56:42.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:42 smithi181 ceph-mon[101970]: failed to encode map e275 with expected crc 2024-03-29T04:56:42.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:42 smithi181 ceph-mon[101970]: failed to encode map e275 with expected crc 2024-03-29T04:56:42.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:42 smithi181 ceph-mon[101970]: osdmap e276: 8 total, 8 up, 8 in 2024-03-29T04:56:42.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:42 smithi181 ceph-mon[101970]: failed to encode map e276 with expected crc 2024-03-29T04:56:42.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:42 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:42.430+0000 7fced2a08700 -1 mon.c@2(peon).osd e276 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:42.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:42 smithi181 ceph-mon[41183]: osdmap e275: 8 total, 8 up, 8 in 2024-03-29T04:56:42.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:42 smithi181 ceph-mon[41183]: failed to encode map e275 with expected crc 2024-03-29T04:56:42.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:42 smithi181 ceph-mon[41183]: failed to encode map e275 with expected crc 2024-03-29T04:56:42.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:42 smithi181 ceph-mon[41183]: failed to encode map e275 with expected crc 2024-03-29T04:56:42.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:42 smithi181 ceph-mon[41183]: failed to encode map e275 with expected crc 2024-03-29T04:56:42.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:42 smithi181 ceph-mon[41183]: osdmap e276: 8 total, 8 up, 8 in 2024-03-29T04:56:42.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:42 smithi181 ceph-mon[41183]: failed to encode map e276 with expected crc 2024-03-29T04:56:42.736 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:56:42.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:42 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:42.430+0000 7f1d52566700 -1 mon.b@1(peon).osd e276 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:42.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:42 smithi187 ceph-mon[33595]: osdmap e275: 8 total, 8 up, 8 in 2024-03-29T04:56:42.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:42 smithi187 ceph-mon[33595]: failed to encode map e275 with expected crc 2024-03-29T04:56:42.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:42 smithi187 ceph-mon[33595]: failed to encode map e275 with expected crc 2024-03-29T04:56:42.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:42 smithi187 ceph-mon[33595]: failed to encode map e275 with expected crc 2024-03-29T04:56:42.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:42 smithi187 ceph-mon[33595]: failed to encode map e275 with expected crc 2024-03-29T04:56:42.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:42 smithi187 ceph-mon[33595]: osdmap e276: 8 total, 8 up, 8 in 2024-03-29T04:56:42.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:42 smithi187 ceph-mon[33595]: failed to encode map e276 with expected crc 2024-03-29T04:56:43.671 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.CloneV2Parent (4042 ms) 2024-03-29T04:56:43.671 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.CloneV1ParentAbort 2024-03-29T04:56:43.703 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:56:43.713 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 0~10 2024-03-29T04:56:43.720 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:56:43.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:43 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:43.531+0000 7f1d52566700 -1 mon.b@1(peon).osd e277 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:43.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:43 smithi187 ceph-mon[33595]: failed to encode map e275 with expected crc 2024-03-29T04:56:43.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:43 smithi187 ceph-mon[33595]: pgmap v51: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 421 KiB/s rd, 25 KiB/s wr, 533 op/s 2024-03-29T04:56:43.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:43 smithi187 ceph-mon[33595]: failed to encode map e276 with expected crc 2024-03-29T04:56:43.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:43 smithi187 ceph-mon[33595]: failed to encode map e276 with expected crc 2024-03-29T04:56:43.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:43 smithi187 ceph-mon[33595]: failed to encode map e276 with expected crc 2024-03-29T04:56:43.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:43 smithi187 ceph-mon[33595]: failed to encode map e276 with expected crc 2024-03-29T04:56:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:43 smithi181 ceph-mon[101970]: failed to encode map e275 with expected crc 2024-03-29T04:56:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:43 smithi181 ceph-mon[101970]: pgmap v51: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 421 KiB/s rd, 25 KiB/s wr, 533 op/s 2024-03-29T04:56:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:43 smithi181 ceph-mon[101970]: failed to encode map e276 with expected crc 2024-03-29T04:56:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:43 smithi181 ceph-mon[101970]: failed to encode map e276 with expected crc 2024-03-29T04:56:43.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:43 smithi181 ceph-mon[101970]: failed to encode map e276 with expected crc 2024-03-29T04:56:43.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:43 smithi181 ceph-mon[101970]: failed to encode map e276 with expected crc 2024-03-29T04:56:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:43 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:43.531+0000 7fced2a08700 -1 mon.c@2(peon).osd e277 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:43 smithi181 ceph-mon[41183]: failed to encode map e275 with expected crc 2024-03-29T04:56:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:43 smithi181 ceph-mon[41183]: pgmap v51: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 421 KiB/s rd, 25 KiB/s wr, 533 op/s 2024-03-29T04:56:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:43 smithi181 ceph-mon[41183]: failed to encode map e276 with expected crc 2024-03-29T04:56:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:43 smithi181 ceph-mon[41183]: failed to encode map e276 with expected crc 2024-03-29T04:56:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:43 smithi181 ceph-mon[41183]: failed to encode map e276 with expected crc 2024-03-29T04:56:43.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:43 smithi181 ceph-mon[41183]: failed to encode map e276 with expected crc 2024-03-29T04:56:44.798 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:44 smithi187 ceph-mon[33595]: osdmap e277: 8 total, 8 up, 8 in 2024-03-29T04:56:44.798 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:44 smithi187 ceph-mon[33595]: failed to encode map e277 with expected crc 2024-03-29T04:56:44.798 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:44 smithi187 ceph-mon[33595]: failed to encode map e277 with expected crc 2024-03-29T04:56:44.798 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:44 smithi187 ceph-mon[33595]: failed to encode map e277 with expected crc 2024-03-29T04:56:44.799 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:44 smithi187 ceph-mon[33595]: failed to encode map e277 with expected crc 2024-03-29T04:56:44.799 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:44 smithi187 ceph-mon[33595]: failed to encode map e277 with expected crc 2024-03-29T04:56:44.799 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:44 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:44.545+0000 7f1d52566700 -1 mon.b@1(peon).osd e278 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:44.799 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:44 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 8. 2024-03-29T04:56:44.799 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:44 smithi187 systemd[1]: Stopped Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:56:44.799 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:44 smithi187 systemd[1]: Starting Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:56:44.799 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:44 smithi187 podman[92143]: 2024-03-29 04:56:44.503497542 +0000 UTC m=+0.017294810 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:56:44.799 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:44 smithi187 podman[92143]: 2024-03-29 04:56:44.609217576 +0000 UTC m=+0.123014819 container create 4823b84efbae4c92ee7b4e42a3714a1d4bfde3184bee71b90d91e681c553e2f3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, release=239.1651231664, architecture=x86_64, version=8.5, maintainer=Paul Cuzner , vcs-type=git, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, distribution-scope=public, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-29T04:56:44.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:44 smithi181 ceph-mon[101970]: osdmap e277: 8 total, 8 up, 8 in 2024-03-29T04:56:44.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:44 smithi181 ceph-mon[101970]: failed to encode map e277 with expected crc 2024-03-29T04:56:44.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:44 smithi181 ceph-mon[101970]: failed to encode map e277 with expected crc 2024-03-29T04:56:44.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:44 smithi181 ceph-mon[101970]: failed to encode map e277 with expected crc 2024-03-29T04:56:44.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:44 smithi181 ceph-mon[101970]: failed to encode map e277 with expected crc 2024-03-29T04:56:44.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:44 smithi181 ceph-mon[101970]: failed to encode map e277 with expected crc 2024-03-29T04:56:44.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:44 smithi181 ceph-mon[41183]: osdmap e277: 8 total, 8 up, 8 in 2024-03-29T04:56:44.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:44 smithi181 ceph-mon[41183]: failed to encode map e277 with expected crc 2024-03-29T04:56:44.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:44 smithi181 ceph-mon[41183]: failed to encode map e277 with expected crc 2024-03-29T04:56:44.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:44 smithi181 ceph-mon[41183]: failed to encode map e277 with expected crc 2024-03-29T04:56:44.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:44 smithi181 ceph-mon[41183]: failed to encode map e277 with expected crc 2024-03-29T04:56:44.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:44 smithi181 ceph-mon[41183]: failed to encode map e277 with expected crc 2024-03-29T04:56:44.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:44 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:44.545+0000 7fced2a08700 -1 mon.c@2(peon).osd e278 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:45.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:44 smithi187 podman[92143]: 2024-03-29 04:56:44.911787341 +0000 UTC m=+0.425584586 container init 4823b84efbae4c92ee7b4e42a3714a1d4bfde3184bee71b90d91e681c553e2f3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, vcs-type=git, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, name=ubi8, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, version=8.5, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-03-29T04:56:45.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:44 smithi187 podman[92143]: 2024-03-29 04:56:44.916919691 +0000 UTC m=+0.430716936 container start 4823b84efbae4c92ee7b4e42a3714a1d4bfde3184bee71b90d91e681c553e2f3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , io.buildah.version=1.32.0, io.openshift.tags=base rhel8, vcs-type=git, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc.) 2024-03-29T04:56:45.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:45 smithi187 bash[92143]: 4823b84efbae4c92ee7b4e42a3714a1d4bfde3184bee71b90d91e681c553e2f3 2024-03-29T04:56:45.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:45 smithi187 systemd[1]: Started Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:56:45.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:45 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a[92157]: logger=settings t=2024-03-29T04:56:45.067735006Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:56:45.383 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:45 smithi187 podman[92193]: 2024-03-29 04:56:45.092825209 +0000 UTC m=+0.018290854 container died 4823b84efbae4c92ee7b4e42a3714a1d4bfde3184bee71b90d91e681c553e2f3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, io.openshift.expose-services=, name=ubi8, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.buildah.version=1.32.0, vendor=Red Hat, Inc., vcs-type=git, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-29T04:56:45.577 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap1 2024-03-29T04:56:45.633 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:56:45.738 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:45 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:45.547+0000 7f1d52566700 -1 mon.b@1(peon).osd e279 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:45.739 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:45 smithi187 ceph-mon[33595]: pgmap v54: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 421 KiB/s rd, 25 KiB/s wr, 533 op/s 2024-03-29T04:56:45.739 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:45 smithi187 ceph-mon[33595]: osdmap e278: 8 total, 8 up, 8 in 2024-03-29T04:56:45.739 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:45 smithi187 ceph-mon[33595]: failed to encode map e278 with expected crc 2024-03-29T04:56:45.739 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:45 smithi187 ceph-mon[33595]: failed to encode map e278 with expected crc 2024-03-29T04:56:45.739 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:45 smithi187 ceph-mon[33595]: failed to encode map e278 with expected crc 2024-03-29T04:56:45.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:45 smithi181 ceph-mon[101970]: pgmap v54: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 421 KiB/s rd, 25 KiB/s wr, 533 op/s 2024-03-29T04:56:45.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:45 smithi181 ceph-mon[101970]: osdmap e278: 8 total, 8 up, 8 in 2024-03-29T04:56:45.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:45 smithi181 ceph-mon[101970]: failed to encode map e278 with expected crc 2024-03-29T04:56:45.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:45 smithi181 ceph-mon[101970]: failed to encode map e278 with expected crc 2024-03-29T04:56:45.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:45 smithi181 ceph-mon[101970]: failed to encode map e278 with expected crc 2024-03-29T04:56:45.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:45 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:45.547+0000 7fced2a08700 -1 mon.c@2(peon).osd e279 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:45.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:45 smithi181 ceph-mon[41183]: pgmap v54: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 421 KiB/s rd, 25 KiB/s wr, 533 op/s 2024-03-29T04:56:45.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:45 smithi181 ceph-mon[41183]: osdmap e278: 8 total, 8 up, 8 in 2024-03-29T04:56:45.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:45 smithi181 ceph-mon[41183]: failed to encode map e278 with expected crc 2024-03-29T04:56:45.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:45 smithi181 ceph-mon[41183]: failed to encode map e278 with expected crc 2024-03-29T04:56:45.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:45 smithi181 ceph-mon[41183]: failed to encode map e278 with expected crc 2024-03-29T04:56:46.132 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:45 smithi187 podman[92193]: 2024-03-29 04:56:45.732780008 +0000 UTC m=+0.658245640 container remove 4823b84efbae4c92ee7b4e42a3714a1d4bfde3184bee71b90d91e681c553e2f3 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-grafana-a, io.buildah.version=1.32.0, maintainer=Paul Cuzner , architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, release=239.1651231664, vcs-type=git, description=Ceph Grafana Container, com.redhat.component=ubi8-container, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., version=8.5, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-03-29T04:56:46.132 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:45 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:56:46.133 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:45 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:56:46.753 INFO:tasks.workunit.client.0.smithi181.stdout:migration_abort 2024-03-29T04:56:46.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:46 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:46.562+0000 7f1d52566700 -1 mon.b@1(peon).osd e280 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:46.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:46 smithi187 ceph-mon[33595]: osdmap e279: 8 total, 8 up, 8 in 2024-03-29T04:56:46.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:46 smithi187 ceph-mon[33595]: failed to encode map e279 with expected crc 2024-03-29T04:56:46.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:46 smithi187 ceph-mon[33595]: failed to encode map e279 with expected crc 2024-03-29T04:56:46.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:46 smithi187 ceph-mon[33595]: failed to encode map e279 with expected crc 2024-03-29T04:56:46.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:46 smithi187 ceph-mon[33595]: failed to encode map e279 with expected crc 2024-03-29T04:56:46.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:46 smithi181 ceph-mon[101970]: osdmap e279: 8 total, 8 up, 8 in 2024-03-29T04:56:46.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:46 smithi181 ceph-mon[101970]: failed to encode map e279 with expected crc 2024-03-29T04:56:46.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:46 smithi181 ceph-mon[101970]: failed to encode map e279 with expected crc 2024-03-29T04:56:46.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:46 smithi181 ceph-mon[101970]: failed to encode map e279 with expected crc 2024-03-29T04:56:46.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:46 smithi181 ceph-mon[101970]: failed to encode map e279 with expected crc 2024-03-29T04:56:46.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:46 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:46.560+0000 7fced2a08700 -1 mon.c@2(peon).osd e280 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:46.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:46 smithi181 ceph-mon[41183]: osdmap e279: 8 total, 8 up, 8 in 2024-03-29T04:56:46.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:46 smithi181 ceph-mon[41183]: failed to encode map e279 with expected crc 2024-03-29T04:56:46.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:46 smithi181 ceph-mon[41183]: failed to encode map e279 with expected crc 2024-03-29T04:56:46.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:46 smithi181 ceph-mon[41183]: failed to encode map e279 with expected crc 2024-03-29T04:56:46.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:46 smithi181 ceph-mon[41183]: failed to encode map e279 with expected crc 2024-03-29T04:56:47.702 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.CloneV1ParentAbort (4031 ms) 2024-03-29T04:56:47.702 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.CloneV2ParentAbort 2024-03-29T04:56:47.733 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:56:47.743 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 0~10 2024-03-29T04:56:47.752 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:56:47.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:47 smithi187 ceph-mon[33595]: pgmap v57: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:56:47.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:47 smithi187 ceph-mon[33595]: osdmap e280: 8 total, 8 up, 8 in 2024-03-29T04:56:47.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:47 smithi187 ceph-mon[33595]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:47 smithi187 ceph-mon[33595]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:47 smithi187 ceph-mon[33595]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:47 smithi187 ceph-mon[33595]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:47 smithi187 ceph-mon[33595]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:47 smithi187 ceph-mon[33595]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:47 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:47.566+0000 7f1d52566700 -1 mon.b@1(peon).osd e281 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:47.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-mon[101970]: pgmap v57: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:56:47.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-mon[101970]: osdmap e280: 8 total, 8 up, 8 in 2024-03-29T04:56:47.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-mon[101970]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-mon[101970]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-mon[101970]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-mon[101970]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-mon[101970]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-mon[101970]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-mon[41183]: pgmap v57: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:56:47.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-mon[41183]: osdmap e280: 8 total, 8 up, 8 in 2024-03-29T04:56:47.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-mon[41183]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-mon[41183]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-mon[41183]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-mon[41183]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-mon[41183]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-mon[41183]: failed to encode map e280 with expected crc 2024-03-29T04:56:47.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:47 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:47.566+0000 7fced2a08700 -1 mon.c@2(peon).osd e281 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:48.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:48.573+0000 7f1d52566700 -1 mon.b@1(peon).osd e282 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:48.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:48 smithi187 ceph-mon[33595]: failed to encode map e281 with expected crc 2024-03-29T04:56:48.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:48 smithi187 ceph-mon[33595]: failed to encode map e281 with expected crc 2024-03-29T04:56:48.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:48 smithi187 ceph-mon[33595]: osdmap e281: 8 total, 8 up, 8 in 2024-03-29T04:56:48.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:48 smithi187 ceph-mon[33595]: failed to encode map e281 with expected crc 2024-03-29T04:56:48.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:48 smithi187 ceph-mon[33595]: failed to encode map e281 with expected crc 2024-03-29T04:56:48.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:56:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:56:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:56:48.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:48 smithi181 ceph-mon[101970]: failed to encode map e281 with expected crc 2024-03-29T04:56:48.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:48 smithi181 ceph-mon[101970]: failed to encode map e281 with expected crc 2024-03-29T04:56:48.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:48 smithi181 ceph-mon[101970]: osdmap e281: 8 total, 8 up, 8 in 2024-03-29T04:56:48.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:48 smithi181 ceph-mon[101970]: failed to encode map e281 with expected crc 2024-03-29T04:56:48.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:48 smithi181 ceph-mon[101970]: failed to encode map e281 with expected crc 2024-03-29T04:56:48.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:48.571+0000 7fced2a08700 -1 mon.c@2(peon).osd e282 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:48.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:48 smithi181 ceph-mon[41183]: failed to encode map e281 with expected crc 2024-03-29T04:56:48.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:48 smithi181 ceph-mon[41183]: failed to encode map e281 with expected crc 2024-03-29T04:56:48.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:48 smithi181 ceph-mon[41183]: osdmap e281: 8 total, 8 up, 8 in 2024-03-29T04:56:48.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:48 smithi181 ceph-mon[41183]: failed to encode map e281 with expected crc 2024-03-29T04:56:48.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:48 smithi181 ceph-mon[41183]: failed to encode map e281 with expected crc 2024-03-29T04:56:49.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:56:48] "GET /metrics HTTP/1.1" 200 38732 "" "Prometheus/2.43.0" 2024-03-29T04:56:49.604 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap1 2024-03-29T04:56:49.669 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:56:49.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:49 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:49.573+0000 7f1d52566700 -1 mon.b@1(peon).osd e283 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:49.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:49 smithi187 ceph-mon[33595]: pgmap v60: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 464 KiB/s rd, 25 KiB/s wr, 579 op/s 2024-03-29T04:56:49.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:49 smithi187 ceph-mon[33595]: osdmap e282: 8 total, 8 up, 8 in 2024-03-29T04:56:49.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:49 smithi187 ceph-mon[33595]: failed to encode map e282 with expected crc 2024-03-29T04:56:49.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:49 smithi187 ceph-mon[33595]: failed to encode map e282 with expected crc 2024-03-29T04:56:49.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:49 smithi181 ceph-mon[101970]: pgmap v60: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 464 KiB/s rd, 25 KiB/s wr, 579 op/s 2024-03-29T04:56:49.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:49 smithi181 ceph-mon[101970]: osdmap e282: 8 total, 8 up, 8 in 2024-03-29T04:56:49.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:49 smithi181 ceph-mon[101970]: failed to encode map e282 with expected crc 2024-03-29T04:56:49.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:49 smithi181 ceph-mon[101970]: failed to encode map e282 with expected crc 2024-03-29T04:56:49.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:49 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:49.573+0000 7fced2a08700 -1 mon.c@2(peon).osd e283 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:49.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:49 smithi181 ceph-mon[41183]: pgmap v60: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 464 KiB/s rd, 25 KiB/s wr, 579 op/s 2024-03-29T04:56:49.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:49 smithi181 ceph-mon[41183]: osdmap e282: 8 total, 8 up, 8 in 2024-03-29T04:56:49.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:49 smithi181 ceph-mon[41183]: failed to encode map e282 with expected crc 2024-03-29T04:56:49.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:49 smithi181 ceph-mon[41183]: failed to encode map e282 with expected crc 2024-03-29T04:56:50.766 INFO:tasks.workunit.client.0.smithi181.stdout:migration_abort 2024-03-29T04:56:50.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:50 smithi187 ceph-mon[33595]: osdmap e283: 8 total, 8 up, 8 in 2024-03-29T04:56:50.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:50 smithi187 ceph-mon[33595]: failed to encode map e283 with expected crc 2024-03-29T04:56:50.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:50 smithi187 ceph-mon[33595]: failed to encode map e283 with expected crc 2024-03-29T04:56:50.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:50 smithi187 ceph-mon[33595]: failed to encode map e283 with expected crc 2024-03-29T04:56:50.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:50 smithi187 ceph-mon[33595]: failed to encode map e283 with expected crc 2024-03-29T04:56:50.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:50 smithi187 ceph-mon[33595]: failed to encode map e283 with expected crc 2024-03-29T04:56:50.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:50 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:50.587+0000 7f1d52566700 -1 mon.b@1(peon).osd e284 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:50.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:50 smithi181 ceph-mon[101970]: osdmap e283: 8 total, 8 up, 8 in 2024-03-29T04:56:50.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:50 smithi181 ceph-mon[101970]: failed to encode map e283 with expected crc 2024-03-29T04:56:50.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:50 smithi181 ceph-mon[101970]: failed to encode map e283 with expected crc 2024-03-29T04:56:50.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:50 smithi181 ceph-mon[101970]: failed to encode map e283 with expected crc 2024-03-29T04:56:50.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:50 smithi181 ceph-mon[101970]: failed to encode map e283 with expected crc 2024-03-29T04:56:50.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:50 smithi181 ceph-mon[101970]: failed to encode map e283 with expected crc 2024-03-29T04:56:50.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:50 smithi181 ceph-mon[41183]: osdmap e283: 8 total, 8 up, 8 in 2024-03-29T04:56:50.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:50 smithi181 ceph-mon[41183]: failed to encode map e283 with expected crc 2024-03-29T04:56:50.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:50 smithi181 ceph-mon[41183]: failed to encode map e283 with expected crc 2024-03-29T04:56:50.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:50 smithi181 ceph-mon[41183]: failed to encode map e283 with expected crc 2024-03-29T04:56:50.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:50 smithi181 ceph-mon[41183]: failed to encode map e283 with expected crc 2024-03-29T04:56:50.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:50 smithi181 ceph-mon[41183]: failed to encode map e283 with expected crc 2024-03-29T04:56:50.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:50 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:50.587+0000 7fced2a08700 -1 mon.c@2(peon).osd e284 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:51.730 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.CloneV2ParentAbort (4029 ms) 2024-03-29T04:56:51.731 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach 2024-03-29T04:56:51.763 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:56:51.774 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 0~10 2024-03-29T04:56:51.782 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:56:51.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:51 smithi187 ceph-mon[33595]: pgmap v63: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 464 KiB/s rd, 25 KiB/s wr, 579 op/s 2024-03-29T04:56:51.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:51 smithi187 ceph-mon[33595]: osdmap e284: 8 total, 8 up, 8 in 2024-03-29T04:56:51.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:51 smithi187 ceph-mon[33595]: failed to encode map e284 with expected crc 2024-03-29T04:56:51.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:51 smithi187 ceph-mon[33595]: failed to encode map e284 with expected crc 2024-03-29T04:56:51.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:51 smithi187 ceph-mon[33595]: failed to encode map e284 with expected crc 2024-03-29T04:56:51.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:51 smithi187 ceph-mon[33595]: failed to encode map e284 with expected crc 2024-03-29T04:56:51.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:51.593+0000 7f1d52566700 -1 mon.b@1(peon).osd e285 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:51.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:51 smithi181 ceph-mon[101970]: pgmap v63: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 464 KiB/s rd, 25 KiB/s wr, 579 op/s 2024-03-29T04:56:51.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:51 smithi181 ceph-mon[101970]: osdmap e284: 8 total, 8 up, 8 in 2024-03-29T04:56:51.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:51 smithi181 ceph-mon[101970]: failed to encode map e284 with expected crc 2024-03-29T04:56:51.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:51 smithi181 ceph-mon[101970]: failed to encode map e284 with expected crc 2024-03-29T04:56:51.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:51 smithi181 ceph-mon[101970]: failed to encode map e284 with expected crc 2024-03-29T04:56:51.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:51 smithi181 ceph-mon[101970]: failed to encode map e284 with expected crc 2024-03-29T04:56:51.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:51 smithi181 ceph-mon[41183]: pgmap v63: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 464 KiB/s rd, 25 KiB/s wr, 579 op/s 2024-03-29T04:56:51.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:51 smithi181 ceph-mon[41183]: osdmap e284: 8 total, 8 up, 8 in 2024-03-29T04:56:51.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:51 smithi181 ceph-mon[41183]: failed to encode map e284 with expected crc 2024-03-29T04:56:51.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:51 smithi181 ceph-mon[41183]: failed to encode map e284 with expected crc 2024-03-29T04:56:51.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:51 smithi181 ceph-mon[41183]: failed to encode map e284 with expected crc 2024-03-29T04:56:51.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:51 smithi181 ceph-mon[41183]: failed to encode map e284 with expected crc 2024-03-29T04:56:51.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:51.593+0000 7fced2a08700 -1 mon.c@2(peon).osd e285 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:52.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:52 smithi181 ceph-mon[101970]: osdmap e285: 8 total, 8 up, 8 in 2024-03-29T04:56:52.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:52 smithi181 ceph-mon[101970]: failed to encode map e285 with expected crc 2024-03-29T04:56:52.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:52 smithi181 ceph-mon[101970]: failed to encode map e285 with expected crc 2024-03-29T04:56:52.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:52 smithi181 ceph-mon[101970]: failed to encode map e285 with expected crc 2024-03-29T04:56:52.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:52 smithi181 ceph-mon[101970]: osdmap e286: 8 total, 8 up, 8 in 2024-03-29T04:56:52.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:52 smithi181 ceph-mon[101970]: failed to encode map e286 with expected crc 2024-03-29T04:56:52.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:52 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:52.434+0000 7fced2a08700 -1 mon.c@2(peon).osd e286 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:52.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:52 smithi181 ceph-mon[41183]: osdmap e285: 8 total, 8 up, 8 in 2024-03-29T04:56:52.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:52 smithi181 ceph-mon[41183]: failed to encode map e285 with expected crc 2024-03-29T04:56:52.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:52 smithi181 ceph-mon[41183]: failed to encode map e285 with expected crc 2024-03-29T04:56:52.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:52 smithi181 ceph-mon[41183]: failed to encode map e285 with expected crc 2024-03-29T04:56:52.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:52 smithi181 ceph-mon[41183]: osdmap e286: 8 total, 8 up, 8 in 2024-03-29T04:56:52.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:52 smithi181 ceph-mon[41183]: failed to encode map e286 with expected crc 2024-03-29T04:56:52.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:52.432+0000 7f1d52566700 -1 mon.b@1(peon).osd e286 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:52.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:52 smithi187 ceph-mon[33595]: osdmap e285: 8 total, 8 up, 8 in 2024-03-29T04:56:52.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:52 smithi187 ceph-mon[33595]: failed to encode map e285 with expected crc 2024-03-29T04:56:52.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:52 smithi187 ceph-mon[33595]: failed to encode map e285 with expected crc 2024-03-29T04:56:52.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:52 smithi187 ceph-mon[33595]: failed to encode map e285 with expected crc 2024-03-29T04:56:52.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:52 smithi187 ceph-mon[33595]: osdmap e286: 8 total, 8 up, 8 in 2024-03-29T04:56:52.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:52 smithi187 ceph-mon[33595]: failed to encode map e286 with expected crc 2024-03-29T04:56:53.472 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap1 2024-03-29T04:56:53.529 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:56:53.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:53 smithi181 ceph-mon[101970]: pgmap v66: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 284 KiB/s rd, 26 KiB/s wr, 378 op/s 2024-03-29T04:56:53.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:53 smithi181 ceph-mon[101970]: failed to encode map e286 with expected crc 2024-03-29T04:56:53.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:53 smithi181 ceph-mon[101970]: failed to encode map e286 with expected crc 2024-03-29T04:56:53.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:53 smithi181 ceph-mon[101970]: failed to encode map e286 with expected crc 2024-03-29T04:56:53.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:53 smithi181 ceph-mon[101970]: osdmap e287: 8 total, 8 up, 8 in 2024-03-29T04:56:53.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:53 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:53.443+0000 7fced2a08700 -1 mon.c@2(peon).osd e287 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:53.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:53 smithi181 ceph-mon[41183]: pgmap v66: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 284 KiB/s rd, 26 KiB/s wr, 378 op/s 2024-03-29T04:56:53.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:53 smithi181 ceph-mon[41183]: failed to encode map e286 with expected crc 2024-03-29T04:56:53.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:53 smithi181 ceph-mon[41183]: failed to encode map e286 with expected crc 2024-03-29T04:56:53.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:53 smithi181 ceph-mon[41183]: failed to encode map e286 with expected crc 2024-03-29T04:56:53.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:53 smithi181 ceph-mon[41183]: osdmap e287: 8 total, 8 up, 8 in 2024-03-29T04:56:53.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:53 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:53.443+0000 7f1d52566700 -1 mon.b@1(peon).osd e287 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:53.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:53 smithi187 ceph-mon[33595]: pgmap v66: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 284 KiB/s rd, 26 KiB/s wr, 378 op/s 2024-03-29T04:56:53.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:53 smithi187 ceph-mon[33595]: failed to encode map e286 with expected crc 2024-03-29T04:56:53.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:53 smithi187 ceph-mon[33595]: failed to encode map e286 with expected crc 2024-03-29T04:56:53.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:53 smithi187 ceph-mon[33595]: failed to encode map e286 with expected crc 2024-03-29T04:56:53.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:53 smithi187 ceph-mon[33595]: osdmap e287: 8 total, 8 up, 8 in 2024-03-29T04:56:54.648 INFO:tasks.workunit.client.0.smithi181.stdout:migration_abort 2024-03-29T04:56:54.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:54 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:54.445+0000 7f1d52566700 -1 mon.b@1(peon).osd e288 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:54.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:54 smithi187 ceph-mon[33595]: failed to encode map e287 with expected crc 2024-03-29T04:56:54.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:54 smithi187 ceph-mon[33595]: failed to encode map e287 with expected crc 2024-03-29T04:56:54.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:54 smithi187 ceph-mon[33595]: failed to encode map e287 with expected crc 2024-03-29T04:56:54.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:54 smithi187 ceph-mon[33595]: failed to encode map e287 with expected crc 2024-03-29T04:56:54.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:54 smithi187 ceph-mon[33595]: failed to encode map e288 with expected crc 2024-03-29T04:56:54.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:54 smithi187 ceph-mon[33595]: osdmap e288: 8 total, 8 up, 8 in 2024-03-29T04:56:54.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:54 smithi181 ceph-mon[101970]: failed to encode map e287 with expected crc 2024-03-29T04:56:54.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:54 smithi181 ceph-mon[101970]: failed to encode map e287 with expected crc 2024-03-29T04:56:54.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:54 smithi181 ceph-mon[101970]: failed to encode map e287 with expected crc 2024-03-29T04:56:54.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:54 smithi181 ceph-mon[101970]: failed to encode map e287 with expected crc 2024-03-29T04:56:54.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:54 smithi181 ceph-mon[101970]: failed to encode map e288 with expected crc 2024-03-29T04:56:54.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:54 smithi181 ceph-mon[101970]: osdmap e288: 8 total, 8 up, 8 in 2024-03-29T04:56:54.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:54 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:54.445+0000 7fced2a08700 -1 mon.c@2(peon).osd e288 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:54.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:54 smithi181 ceph-mon[41183]: failed to encode map e287 with expected crc 2024-03-29T04:56:54.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:54 smithi181 ceph-mon[41183]: failed to encode map e287 with expected crc 2024-03-29T04:56:54.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:54 smithi181 ceph-mon[41183]: failed to encode map e287 with expected crc 2024-03-29T04:56:54.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:54 smithi181 ceph-mon[41183]: failed to encode map e287 with expected crc 2024-03-29T04:56:54.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:54 smithi181 ceph-mon[41183]: failed to encode map e288 with expected crc 2024-03-29T04:56:54.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:54 smithi181 ceph-mon[41183]: osdmap e288: 8 total, 8 up, 8 in 2024-03-29T04:56:55.749 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach (4018 ms) 2024-03-29T04:56:55.749 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixParentReattach 2024-03-29T04:56:55.778 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:56:55.789 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 0~10 2024-03-29T04:56:55.797 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:56:55.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:55 smithi187 ceph-mon[33595]: pgmap v69: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 284 KiB/s rd, 26 KiB/s wr, 378 op/s 2024-03-29T04:56:55.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:55 smithi187 ceph-mon[33595]: failed to encode map e288 with expected crc 2024-03-29T04:56:55.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:55 smithi187 ceph-mon[33595]: failed to encode map e288 with expected crc 2024-03-29T04:56:55.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:55 smithi187 ceph-mon[33595]: failed to encode map e288 with expected crc 2024-03-29T04:56:55.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:55.614+0000 7f1d52566700 -1 mon.b@1(peon).osd e289 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:55.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:55 smithi181 ceph-mon[101970]: pgmap v69: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 284 KiB/s rd, 26 KiB/s wr, 378 op/s 2024-03-29T04:56:55.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:55 smithi181 ceph-mon[101970]: failed to encode map e288 with expected crc 2024-03-29T04:56:55.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:55 smithi181 ceph-mon[101970]: failed to encode map e288 with expected crc 2024-03-29T04:56:55.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:55 smithi181 ceph-mon[101970]: failed to encode map e288 with expected crc 2024-03-29T04:56:55.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:55 smithi181 ceph-mon[41183]: pgmap v69: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 284 KiB/s rd, 26 KiB/s wr, 378 op/s 2024-03-29T04:56:55.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:55 smithi181 ceph-mon[41183]: failed to encode map e288 with expected crc 2024-03-29T04:56:55.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:55 smithi181 ceph-mon[41183]: failed to encode map e288 with expected crc 2024-03-29T04:56:55.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:55 smithi181 ceph-mon[41183]: failed to encode map e288 with expected crc 2024-03-29T04:56:55.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:55 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:55.612+0000 7fced2a08700 -1 mon.c@2(peon).osd e289 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:56.625 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:56 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 9. 2024-03-29T04:56:56.625 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:56 smithi187 systemd[1]: Stopped Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:56:56.625 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:56 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Start request repeated too quickly. 2024-03-29T04:56:56.625 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:56 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:56:56.625 INFO:journalctl@ceph.grafana.a.smithi187.stdout:Mar 29 04:56:56 smithi187 systemd[1]: Failed to start Ceph grafana.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:56:56.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:56.617+0000 7f1d52566700 -1 mon.b@1(peon).osd e290 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:56.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:56 smithi187 ceph-mon[33595]: osdmap e289: 8 total, 8 up, 8 in 2024-03-29T04:56:56.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:56 smithi187 ceph-mon[33595]: failed to encode map e289 with expected crc 2024-03-29T04:56:56.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:56 smithi187 ceph-mon[33595]: failed to encode map e289 with expected crc 2024-03-29T04:56:56.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:56 smithi187 ceph-mon[33595]: failed to encode map e289 with expected crc 2024-03-29T04:56:56.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:56 smithi187 ceph-mon[33595]: failed to encode map e289 with expected crc 2024-03-29T04:56:56.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:56 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:56.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:56 smithi181 ceph-mon[101970]: osdmap e289: 8 total, 8 up, 8 in 2024-03-29T04:56:56.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:56 smithi181 ceph-mon[101970]: failed to encode map e289 with expected crc 2024-03-29T04:56:56.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:56 smithi181 ceph-mon[101970]: failed to encode map e289 with expected crc 2024-03-29T04:56:56.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:56 smithi181 ceph-mon[101970]: failed to encode map e289 with expected crc 2024-03-29T04:56:56.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:56 smithi181 ceph-mon[101970]: failed to encode map e289 with expected crc 2024-03-29T04:56:56.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:56.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:56 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:56.618+0000 7fced2a08700 -1 mon.c@2(peon).osd e290 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:56.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:56 smithi181 ceph-mon[41183]: osdmap e289: 8 total, 8 up, 8 in 2024-03-29T04:56:56.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:56 smithi181 ceph-mon[41183]: failed to encode map e289 with expected crc 2024-03-29T04:56:56.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:56 smithi181 ceph-mon[41183]: failed to encode map e289 with expected crc 2024-03-29T04:56:56.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:56 smithi181 ceph-mon[41183]: failed to encode map e289 with expected crc 2024-03-29T04:56:56.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:56 smithi181 ceph-mon[41183]: failed to encode map e289 with expected crc 2024-03-29T04:56:56.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:56 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:57.654 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap1 2024-03-29T04:56:57.713 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:56:57.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:57 smithi187 ceph-mon[33595]: pgmap v72: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:56:57.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:57 smithi187 ceph-mon[33595]: osdmap e290: 8 total, 8 up, 8 in 2024-03-29T04:56:57.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:57 smithi187 ceph-mon[33595]: failed to encode map e290 with expected crc 2024-03-29T04:56:57.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:57 smithi187 ceph-mon[33595]: failed to encode map e290 with expected crc 2024-03-29T04:56:57.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:57 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:57.624+0000 7f1d52566700 -1 mon.b@1(peon).osd e291 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:57.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:57 smithi181 ceph-mon[101970]: pgmap v72: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:56:57.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:57 smithi181 ceph-mon[101970]: osdmap e290: 8 total, 8 up, 8 in 2024-03-29T04:56:57.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:57 smithi181 ceph-mon[101970]: failed to encode map e290 with expected crc 2024-03-29T04:56:57.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:57 smithi181 ceph-mon[101970]: failed to encode map e290 with expected crc 2024-03-29T04:56:57.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:57 smithi181 ceph-mon[41183]: pgmap v72: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:56:57.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:57 smithi181 ceph-mon[41183]: osdmap e290: 8 total, 8 up, 8 in 2024-03-29T04:56:57.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:57 smithi181 ceph-mon[41183]: failed to encode map e290 with expected crc 2024-03-29T04:56:57.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:57 smithi181 ceph-mon[41183]: failed to encode map e290 with expected crc 2024-03-29T04:56:57.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:57 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:57.624+0000 7fced2a08700 -1 mon.c@2(peon).osd e291 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:58.844 INFO:tasks.workunit.client.0.smithi181.stdout:migration_abort 2024-03-29T04:56:58.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:58 smithi181 ceph-mon[101970]: osdmap e291: 8 total, 8 up, 8 in 2024-03-29T04:56:58.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:58 smithi181 ceph-mon[101970]: failed to encode map e291 with expected crc 2024-03-29T04:56:58.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:58 smithi181 ceph-mon[101970]: failed to encode map e291 with expected crc 2024-03-29T04:56:58.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:58 smithi181 ceph-mon[101970]: failed to encode map e291 with expected crc 2024-03-29T04:56:58.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:58 smithi181 ceph-mon[101970]: failed to encode map e291 with expected crc 2024-03-29T04:56:58.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:58 smithi181 ceph-mon[101970]: failed to encode map e291 with expected crc 2024-03-29T04:56:58.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:58.635+0000 7fced2a08700 -1 mon.c@2(peon).osd e292 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:58.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:58 smithi181 ceph-mon[41183]: osdmap e291: 8 total, 8 up, 8 in 2024-03-29T04:56:58.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:58 smithi181 ceph-mon[41183]: failed to encode map e291 with expected crc 2024-03-29T04:56:58.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:58 smithi181 ceph-mon[41183]: failed to encode map e291 with expected crc 2024-03-29T04:56:58.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:58 smithi181 ceph-mon[41183]: failed to encode map e291 with expected crc 2024-03-29T04:56:58.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:58 smithi181 ceph-mon[41183]: failed to encode map e291 with expected crc 2024-03-29T04:56:58.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:58 smithi181 ceph-mon[41183]: failed to encode map e291 with expected crc 2024-03-29T04:56:59.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:58 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:58.632+0000 7f1d52566700 -1 mon.b@1(peon).osd e292 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:56:59.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:58 smithi187 ceph-mon[33595]: osdmap e291: 8 total, 8 up, 8 in 2024-03-29T04:56:59.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:58 smithi187 ceph-mon[33595]: failed to encode map e291 with expected crc 2024-03-29T04:56:59.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:58 smithi187 ceph-mon[33595]: failed to encode map e291 with expected crc 2024-03-29T04:56:59.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:58 smithi187 ceph-mon[33595]: failed to encode map e291 with expected crc 2024-03-29T04:56:59.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:58 smithi187 ceph-mon[33595]: failed to encode map e291 with expected crc 2024-03-29T04:56:59.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:58 smithi187 ceph-mon[33595]: failed to encode map e291 with expected crc 2024-03-29T04:56:59.133 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:56:58 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:56:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:56:59.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:56:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:56:58] "GET /metrics HTTP/1.1" 200 38727 "" "Prometheus/2.43.0" 2024-03-29T04:56:59.779 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixParentReattach (4030 ms) 2024-03-29T04:56:59.779 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.CloneV1ParentAbortRelinkNotNeeded 2024-03-29T04:56:59.808 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:56:59.819 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 0~10 2024-03-29T04:56:59.828 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:56:59.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-mon[101970]: pgmap v75: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 281 KiB/s rd, 17 KiB/s wr, 358 op/s 2024-03-29T04:56:59.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-mon[101970]: failed to encode map e292 with expected crc 2024-03-29T04:56:59.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-mon[101970]: osdmap e292: 8 total, 8 up, 8 in 2024-03-29T04:56:59.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-mon[101970]: failed to encode map e292 with expected crc 2024-03-29T04:56:59.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-mon[101970]: failed to encode map e292 with expected crc 2024-03-29T04:56:59.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-mon[101970]: failed to encode map e292 with expected crc 2024-03-29T04:56:59.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-mon[101970]: failed to encode map e292 with expected crc 2024-03-29T04:56:59.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-mon[101970]: failed to encode map e292 with expected crc 2024-03-29T04:56:59.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-mon[41183]: pgmap v75: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 281 KiB/s rd, 17 KiB/s wr, 358 op/s 2024-03-29T04:56:59.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-mon[41183]: failed to encode map e292 with expected crc 2024-03-29T04:56:59.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-mon[41183]: osdmap e292: 8 total, 8 up, 8 in 2024-03-29T04:56:59.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-mon[41183]: failed to encode map e292 with expected crc 2024-03-29T04:56:59.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-mon[41183]: failed to encode map e292 with expected crc 2024-03-29T04:56:59.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-mon[41183]: failed to encode map e292 with expected crc 2024-03-29T04:56:59.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-mon[41183]: failed to encode map e292 with expected crc 2024-03-29T04:56:59.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-mon[41183]: failed to encode map e292 with expected crc 2024-03-29T04:56:59.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:56:59 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:56:59.640+0000 7fced2a08700 -1 mon.c@2(peon).osd e293 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:00.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:59 smithi187 ceph-mon[33595]: pgmap v75: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 281 KiB/s rd, 17 KiB/s wr, 358 op/s 2024-03-29T04:57:00.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:59 smithi187 ceph-mon[33595]: failed to encode map e292 with expected crc 2024-03-29T04:57:00.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:59 smithi187 ceph-mon[33595]: osdmap e292: 8 total, 8 up, 8 in 2024-03-29T04:57:00.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:59 smithi187 ceph-mon[33595]: failed to encode map e292 with expected crc 2024-03-29T04:57:00.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:59 smithi187 ceph-mon[33595]: failed to encode map e292 with expected crc 2024-03-29T04:57:00.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:59 smithi187 ceph-mon[33595]: failed to encode map e292 with expected crc 2024-03-29T04:57:00.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:59 smithi187 ceph-mon[33595]: failed to encode map e292 with expected crc 2024-03-29T04:57:00.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:59 smithi187 ceph-mon[33595]: failed to encode map e292 with expected crc 2024-03-29T04:57:00.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:56:59 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:56:59.638+0000 7f1d52566700 -1 mon.b@1(peon).osd e293 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:00.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:00 smithi181 ceph-mon[101970]: failed to encode map e293 with expected crc 2024-03-29T04:57:00.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:00 smithi181 ceph-mon[101970]: failed to encode map e293 with expected crc 2024-03-29T04:57:00.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:00 smithi181 ceph-mon[101970]: osdmap e293: 8 total, 8 up, 8 in 2024-03-29T04:57:00.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:00 smithi181 ceph-mon[101970]: failed to encode map e293 with expected crc 2024-03-29T04:57:00.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:00 smithi181 ceph-mon[101970]: failed to encode map e293 with expected crc 2024-03-29T04:57:00.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:00 smithi181 ceph-mon[101970]: failed to encode map e293 with expected crc 2024-03-29T04:57:00.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:00 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:00.649+0000 7fced2a08700 -1 mon.c@2(peon).osd e294 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:00.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:00 smithi181 ceph-mon[41183]: failed to encode map e293 with expected crc 2024-03-29T04:57:00.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:00 smithi181 ceph-mon[41183]: failed to encode map e293 with expected crc 2024-03-29T04:57:00.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:00 smithi181 ceph-mon[41183]: osdmap e293: 8 total, 8 up, 8 in 2024-03-29T04:57:00.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:00 smithi181 ceph-mon[41183]: failed to encode map e293 with expected crc 2024-03-29T04:57:00.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:00 smithi181 ceph-mon[41183]: failed to encode map e293 with expected crc 2024-03-29T04:57:00.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:00 smithi181 ceph-mon[41183]: failed to encode map e293 with expected crc 2024-03-29T04:57:01.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:00 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:00.647+0000 7f1d52566700 -1 mon.b@1(peon).osd e294 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:01.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:00 smithi187 ceph-mon[33595]: failed to encode map e293 with expected crc 2024-03-29T04:57:01.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:00 smithi187 ceph-mon[33595]: failed to encode map e293 with expected crc 2024-03-29T04:57:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:00 smithi187 ceph-mon[33595]: osdmap e293: 8 total, 8 up, 8 in 2024-03-29T04:57:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:00 smithi187 ceph-mon[33595]: failed to encode map e293 with expected crc 2024-03-29T04:57:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:00 smithi187 ceph-mon[33595]: failed to encode map e293 with expected crc 2024-03-29T04:57:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:00 smithi187 ceph-mon[33595]: failed to encode map e293 with expected crc 2024-03-29T04:57:01.685 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap1 2024-03-29T04:57:01.743 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:57:01.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:01 smithi181 ceph-mon[41183]: pgmap v78: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 281 KiB/s rd, 17 KiB/s wr, 358 op/s 2024-03-29T04:57:01.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:01 smithi181 ceph-mon[41183]: failed to encode map e294 with expected crc 2024-03-29T04:57:01.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:01 smithi181 ceph-mon[41183]: osdmap e294: 8 total, 8 up, 8 in 2024-03-29T04:57:01.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:01 smithi181 ceph-mon[41183]: failed to encode map e294 with expected crc 2024-03-29T04:57:01.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:01 smithi181 ceph-mon[41183]: failed to encode map e294 with expected crc 2024-03-29T04:57:01.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:01 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:01.656+0000 7fced2a08700 -1 mon.c@2(peon).osd e295 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:01.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:01 smithi181 ceph-mon[101970]: pgmap v78: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 281 KiB/s rd, 17 KiB/s wr, 358 op/s 2024-03-29T04:57:01.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:01 smithi181 ceph-mon[101970]: failed to encode map e294 with expected crc 2024-03-29T04:57:01.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:01 smithi181 ceph-mon[101970]: osdmap e294: 8 total, 8 up, 8 in 2024-03-29T04:57:01.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:01 smithi181 ceph-mon[101970]: failed to encode map e294 with expected crc 2024-03-29T04:57:01.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:01 smithi181 ceph-mon[101970]: failed to encode map e294 with expected crc 2024-03-29T04:57:02.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:01 smithi187 ceph-mon[33595]: pgmap v78: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 281 KiB/s rd, 17 KiB/s wr, 358 op/s 2024-03-29T04:57:02.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:01 smithi187 ceph-mon[33595]: failed to encode map e294 with expected crc 2024-03-29T04:57:02.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:01 smithi187 ceph-mon[33595]: osdmap e294: 8 total, 8 up, 8 in 2024-03-29T04:57:02.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:01 smithi187 ceph-mon[33595]: failed to encode map e294 with expected crc 2024-03-29T04:57:02.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:01 smithi187 ceph-mon[33595]: failed to encode map e294 with expected crc 2024-03-29T04:57:02.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:01 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:01.656+0000 7f1d52566700 -1 mon.b@1(peon).osd e295 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:02.639 INFO:tasks.workunit.client.0.smithi181.stdout:migration_abort 2024-03-29T04:57:02.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-mon[101970]: failed to encode map e295 with expected crc 2024-03-29T04:57:02.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-mon[101970]: osdmap e295: 8 total, 8 up, 8 in 2024-03-29T04:57:02.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-mon[101970]: failed to encode map e295 with expected crc 2024-03-29T04:57:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-mon[101970]: failed to encode map e295 with expected crc 2024-03-29T04:57:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-mon[101970]: failed to encode map e295 with expected crc 2024-03-29T04:57:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-mon[101970]: osdmap e296: 8 total, 8 up, 8 in 2024-03-29T04:57:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-mon[101970]: failed to encode map e296 with expected crc 2024-03-29T04:57:02.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-mon[101970]: failed to encode map e296 with expected crc 2024-03-29T04:57:02.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:02.435+0000 7fced2a08700 -1 mon.c@2(peon).osd e296 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:02.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-mon[41183]: failed to encode map e295 with expected crc 2024-03-29T04:57:02.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-mon[41183]: osdmap e295: 8 total, 8 up, 8 in 2024-03-29T04:57:02.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-mon[41183]: failed to encode map e295 with expected crc 2024-03-29T04:57:02.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-mon[41183]: failed to encode map e295 with expected crc 2024-03-29T04:57:02.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-mon[41183]: failed to encode map e295 with expected crc 2024-03-29T04:57:02.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-mon[41183]: osdmap e296: 8 total, 8 up, 8 in 2024-03-29T04:57:02.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-mon[41183]: failed to encode map e296 with expected crc 2024-03-29T04:57:02.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:02 smithi181 ceph-mon[41183]: failed to encode map e296 with expected crc 2024-03-29T04:57:02.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:02 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:02.435+0000 7f1d52566700 -1 mon.b@1(peon).osd e296 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:02.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:02 smithi187 ceph-mon[33595]: failed to encode map e295 with expected crc 2024-03-29T04:57:02.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:02 smithi187 ceph-mon[33595]: osdmap e295: 8 total, 8 up, 8 in 2024-03-29T04:57:02.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:02 smithi187 ceph-mon[33595]: failed to encode map e295 with expected crc 2024-03-29T04:57:02.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:02 smithi187 ceph-mon[33595]: failed to encode map e295 with expected crc 2024-03-29T04:57:02.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:02 smithi187 ceph-mon[33595]: failed to encode map e295 with expected crc 2024-03-29T04:57:02.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:02 smithi187 ceph-mon[33595]: osdmap e296: 8 total, 8 up, 8 in 2024-03-29T04:57:02.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:02 smithi187 ceph-mon[33595]: failed to encode map e296 with expected crc 2024-03-29T04:57:02.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:02 smithi187 ceph-mon[33595]: failed to encode map e296 with expected crc 2024-03-29T04:57:03.804 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.CloneV1ParentAbortRelinkNotNeeded (4025 ms) 2024-03-29T04:57:03.804 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach 2024-03-29T04:57:03.835 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:57:03.846 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 0~10 2024-03-29T04:57:03.854 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:57:03.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:03 smithi181 ceph-mon[101970]: pgmap v81: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 351 KiB/s rd, 21 KiB/s wr, 450 op/s 2024-03-29T04:57:03.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:03 smithi181 ceph-mon[101970]: failed to encode map e296 with expected crc 2024-03-29T04:57:03.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:03 smithi181 ceph-mon[41183]: pgmap v81: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 351 KiB/s rd, 21 KiB/s wr, 450 op/s 2024-03-29T04:57:03.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:03 smithi181 ceph-mon[41183]: failed to encode map e296 with expected crc 2024-03-29T04:57:03.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:03 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:03.673+0000 7fced2a08700 -1 mon.c@2(peon).osd e297 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:04.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:03 smithi187 ceph-mon[33595]: pgmap v81: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 351 KiB/s rd, 21 KiB/s wr, 450 op/s 2024-03-29T04:57:04.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:03 smithi187 ceph-mon[33595]: failed to encode map e296 with expected crc 2024-03-29T04:57:04.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:03.673+0000 7f1d52566700 -1 mon.b@1(peon).osd e297 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:04.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:04 smithi181 ceph-mon[101970]: osdmap e297: 8 total, 8 up, 8 in 2024-03-29T04:57:04.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:04 smithi181 ceph-mon[101970]: failed to encode map e297 with expected crc 2024-03-29T04:57:04.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:04 smithi181 ceph-mon[101970]: failed to encode map e297 with expected crc 2024-03-29T04:57:04.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:04 smithi181 ceph-mon[101970]: failed to encode map e297 with expected crc 2024-03-29T04:57:04.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:04 smithi181 ceph-mon[101970]: failed to encode map e297 with expected crc 2024-03-29T04:57:04.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:04 smithi181 ceph-mon[101970]: failed to encode map e297 with expected crc 2024-03-29T04:57:04.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:04 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:04.676+0000 7fced2a08700 -1 mon.c@2(peon).osd e298 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:04.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:04 smithi181 ceph-mon[41183]: osdmap e297: 8 total, 8 up, 8 in 2024-03-29T04:57:04.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:04 smithi181 ceph-mon[41183]: failed to encode map e297 with expected crc 2024-03-29T04:57:04.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:04 smithi181 ceph-mon[41183]: failed to encode map e297 with expected crc 2024-03-29T04:57:04.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:04 smithi181 ceph-mon[41183]: failed to encode map e297 with expected crc 2024-03-29T04:57:04.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:04 smithi181 ceph-mon[41183]: failed to encode map e297 with expected crc 2024-03-29T04:57:04.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:04 smithi181 ceph-mon[41183]: failed to encode map e297 with expected crc 2024-03-29T04:57:05.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:04.676+0000 7f1d52566700 -1 mon.b@1(peon).osd e298 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:05.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:04 smithi187 ceph-mon[33595]: osdmap e297: 8 total, 8 up, 8 in 2024-03-29T04:57:05.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:04 smithi187 ceph-mon[33595]: failed to encode map e297 with expected crc 2024-03-29T04:57:05.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:04 smithi187 ceph-mon[33595]: failed to encode map e297 with expected crc 2024-03-29T04:57:05.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:04 smithi187 ceph-mon[33595]: failed to encode map e297 with expected crc 2024-03-29T04:57:05.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:04 smithi187 ceph-mon[33595]: failed to encode map e297 with expected crc 2024-03-29T04:57:05.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:04 smithi187 ceph-mon[33595]: failed to encode map e297 with expected crc 2024-03-29T04:57:05.694 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:05 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:05.678+0000 7f1d52566700 -1 mon.b@1(peon).osd e299 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:05.695 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:05 smithi187 ceph-mon[33595]: pgmap v84: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 351 KiB/s rd, 21 KiB/s wr, 450 op/s 2024-03-29T04:57:05.695 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:05 smithi187 ceph-mon[33595]: osdmap e298: 8 total, 8 up, 8 in 2024-03-29T04:57:05.695 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:05 smithi187 ceph-mon[33595]: failed to encode map e298 with expected crc 2024-03-29T04:57:05.695 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:05 smithi187 ceph-mon[33595]: failed to encode map e298 with expected crc 2024-03-29T04:57:05.695 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:05 smithi187 ceph-mon[33595]: failed to encode map e298 with expected crc 2024-03-29T04:57:05.708 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap1 2024-03-29T04:57:05.767 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:57:05.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:05 smithi181 ceph-mon[101970]: pgmap v84: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 351 KiB/s rd, 21 KiB/s wr, 450 op/s 2024-03-29T04:57:05.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:05 smithi181 ceph-mon[101970]: osdmap e298: 8 total, 8 up, 8 in 2024-03-29T04:57:05.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:05 smithi181 ceph-mon[101970]: failed to encode map e298 with expected crc 2024-03-29T04:57:05.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:05 smithi181 ceph-mon[101970]: failed to encode map e298 with expected crc 2024-03-29T04:57:05.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:05 smithi181 ceph-mon[101970]: failed to encode map e298 with expected crc 2024-03-29T04:57:05.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:05 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:05.678+0000 7fced2a08700 -1 mon.c@2(peon).osd e299 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:05.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:05 smithi181 ceph-mon[41183]: pgmap v84: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 351 KiB/s rd, 21 KiB/s wr, 450 op/s 2024-03-29T04:57:05.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:05 smithi181 ceph-mon[41183]: osdmap e298: 8 total, 8 up, 8 in 2024-03-29T04:57:05.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:05 smithi181 ceph-mon[41183]: failed to encode map e298 with expected crc 2024-03-29T04:57:05.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:05 smithi181 ceph-mon[41183]: failed to encode map e298 with expected crc 2024-03-29T04:57:05.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:05 smithi181 ceph-mon[41183]: failed to encode map e298 with expected crc 2024-03-29T04:57:06.897 INFO:tasks.workunit.client.0.smithi181.stdout:migration_abort 2024-03-29T04:57:07.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:06 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:06.694+0000 7f1d52566700 -1 mon.b@1(peon).osd e300 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:07.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:06 smithi187 ceph-mon[33595]: osdmap e299: 8 total, 8 up, 8 in 2024-03-29T04:57:07.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:06 smithi187 ceph-mon[33595]: failed to encode map e299 with expected crc 2024-03-29T04:57:07.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:06 smithi187 ceph-mon[33595]: failed to encode map e299 with expected crc 2024-03-29T04:57:07.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:06 smithi187 ceph-mon[33595]: failed to encode map e299 with expected crc 2024-03-29T04:57:07.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:06 smithi187 ceph-mon[33595]: failed to encode map e299 with expected crc 2024-03-29T04:57:07.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:06 smithi181 ceph-mon[101970]: osdmap e299: 8 total, 8 up, 8 in 2024-03-29T04:57:07.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:06 smithi181 ceph-mon[101970]: failed to encode map e299 with expected crc 2024-03-29T04:57:07.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:06 smithi181 ceph-mon[101970]: failed to encode map e299 with expected crc 2024-03-29T04:57:07.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:06 smithi181 ceph-mon[101970]: failed to encode map e299 with expected crc 2024-03-29T04:57:07.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:06 smithi181 ceph-mon[101970]: failed to encode map e299 with expected crc 2024-03-29T04:57:07.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:06 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:06.696+0000 7fced2a08700 -1 mon.c@2(peon).osd e300 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:07.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:06 smithi181 ceph-mon[41183]: osdmap e299: 8 total, 8 up, 8 in 2024-03-29T04:57:07.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:06 smithi181 ceph-mon[41183]: failed to encode map e299 with expected crc 2024-03-29T04:57:07.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:06 smithi181 ceph-mon[41183]: failed to encode map e299 with expected crc 2024-03-29T04:57:07.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:06 smithi181 ceph-mon[41183]: failed to encode map e299 with expected crc 2024-03-29T04:57:07.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:06 smithi181 ceph-mon[41183]: failed to encode map e299 with expected crc 2024-03-29T04:57:07.845 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach (4042 ms) 2024-03-29T04:57:07.846 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixParentReattach 2024-03-29T04:57:07.877 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:57:07.887 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 0~10 2024-03-29T04:57:07.895 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:57:08.104 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:07 smithi181 ceph-mon[101970]: pgmap v87: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:57:08.104 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:07 smithi181 ceph-mon[101970]: failed to encode map e300 with expected crc 2024-03-29T04:57:08.104 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:07 smithi181 ceph-mon[101970]: failed to encode map e300 with expected crc 2024-03-29T04:57:08.104 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:07 smithi181 ceph-mon[101970]: osdmap e300: 8 total, 8 up, 8 in 2024-03-29T04:57:08.104 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:07.695+0000 7fced2a08700 -1 mon.c@2(peon).osd e301 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:08.104 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:07 smithi181 ceph-mon[41183]: pgmap v87: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:57:08.104 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:07 smithi181 ceph-mon[41183]: failed to encode map e300 with expected crc 2024-03-29T04:57:08.105 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:07 smithi181 ceph-mon[41183]: failed to encode map e300 with expected crc 2024-03-29T04:57:08.105 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:07 smithi181 ceph-mon[41183]: osdmap e300: 8 total, 8 up, 8 in 2024-03-29T04:57:08.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:07 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:07.695+0000 7f1d52566700 -1 mon.b@1(peon).osd e301 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:08.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:07 smithi187 ceph-mon[33595]: pgmap v87: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:57:08.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:07 smithi187 ceph-mon[33595]: failed to encode map e300 with expected crc 2024-03-29T04:57:08.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:07 smithi187 ceph-mon[33595]: failed to encode map e300 with expected crc 2024-03-29T04:57:08.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:07 smithi187 ceph-mon[33595]: osdmap e300: 8 total, 8 up, 8 in 2024-03-29T04:57:09.133 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:57:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:57:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:57:09.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:08 smithi187 ceph-mon[33595]: osdmap e301: 8 total, 8 up, 8 in 2024-03-29T04:57:09.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:08 smithi187 ceph-mon[33595]: failed to encode map e301 with expected crc 2024-03-29T04:57:09.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:08 smithi187 ceph-mon[33595]: failed to encode map e301 with expected crc 2024-03-29T04:57:09.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:08 smithi187 ceph-mon[33595]: failed to encode map e301 with expected crc 2024-03-29T04:57:09.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:08 smithi187 ceph-mon[33595]: failed to encode map e301 with expected crc 2024-03-29T04:57:09.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:08 smithi187 ceph-mon[33595]: failed to encode map e301 with expected crc 2024-03-29T04:57:09.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:08.710+0000 7f1d52566700 -1 mon.b@1(peon).osd e302 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:09.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:57:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:57:08] "GET /metrics HTTP/1.1" 200 38727 "" "Prometheus/2.43.0" 2024-03-29T04:57:09.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:08 smithi181 ceph-mon[41183]: osdmap e301: 8 total, 8 up, 8 in 2024-03-29T04:57:09.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:08 smithi181 ceph-mon[41183]: failed to encode map e301 with expected crc 2024-03-29T04:57:09.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:08 smithi181 ceph-mon[41183]: failed to encode map e301 with expected crc 2024-03-29T04:57:09.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:08 smithi181 ceph-mon[41183]: failed to encode map e301 with expected crc 2024-03-29T04:57:09.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:08 smithi181 ceph-mon[41183]: failed to encode map e301 with expected crc 2024-03-29T04:57:09.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:08 smithi181 ceph-mon[41183]: failed to encode map e301 with expected crc 2024-03-29T04:57:09.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:08.713+0000 7fced2a08700 -1 mon.c@2(peon).osd e302 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:09.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:08 smithi181 ceph-mon[101970]: osdmap e301: 8 total, 8 up, 8 in 2024-03-29T04:57:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:08 smithi181 ceph-mon[101970]: failed to encode map e301 with expected crc 2024-03-29T04:57:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:08 smithi181 ceph-mon[101970]: failed to encode map e301 with expected crc 2024-03-29T04:57:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:08 smithi181 ceph-mon[101970]: failed to encode map e301 with expected crc 2024-03-29T04:57:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:08 smithi181 ceph-mon[101970]: failed to encode map e301 with expected crc 2024-03-29T04:57:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:08 smithi181 ceph-mon[101970]: failed to encode map e301 with expected crc 2024-03-29T04:57:09.750 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap1 2024-03-29T04:57:09.811 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:57:10.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:09 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:09.712+0000 7f1d52566700 -1 mon.b@1(peon).osd e303 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:10.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:09 smithi187 ceph-mon[33595]: pgmap v90: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 378 KiB/s rd, 25 KiB/s wr, 479 op/s 2024-03-29T04:57:10.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:09 smithi187 ceph-mon[33595]: failed to encode map e302 with expected crc 2024-03-29T04:57:10.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:09 smithi187 ceph-mon[33595]: osdmap e302: 8 total, 8 up, 8 in 2024-03-29T04:57:10.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:09 smithi187 ceph-mon[33595]: failed to encode map e302 with expected crc 2024-03-29T04:57:10.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:09 smithi187 ceph-mon[33595]: failed to encode map e302 with expected crc 2024-03-29T04:57:10.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:09 smithi187 ceph-mon[33595]: failed to encode map e302 with expected crc 2024-03-29T04:57:10.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:09 smithi181 ceph-mon[101970]: pgmap v90: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 378 KiB/s rd, 25 KiB/s wr, 479 op/s 2024-03-29T04:57:10.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:09 smithi181 ceph-mon[101970]: failed to encode map e302 with expected crc 2024-03-29T04:57:10.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:09 smithi181 ceph-mon[101970]: osdmap e302: 8 total, 8 up, 8 in 2024-03-29T04:57:10.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:09 smithi181 ceph-mon[101970]: failed to encode map e302 with expected crc 2024-03-29T04:57:10.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:09 smithi181 ceph-mon[101970]: failed to encode map e302 with expected crc 2024-03-29T04:57:10.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:09 smithi181 ceph-mon[101970]: failed to encode map e302 with expected crc 2024-03-29T04:57:10.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:09 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:09.715+0000 7fced2a08700 -1 mon.c@2(peon).osd e303 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:10.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:09 smithi181 ceph-mon[41183]: pgmap v90: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 378 KiB/s rd, 25 KiB/s wr, 479 op/s 2024-03-29T04:57:10.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:09 smithi181 ceph-mon[41183]: failed to encode map e302 with expected crc 2024-03-29T04:57:10.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:09 smithi181 ceph-mon[41183]: osdmap e302: 8 total, 8 up, 8 in 2024-03-29T04:57:10.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:09 smithi181 ceph-mon[41183]: failed to encode map e302 with expected crc 2024-03-29T04:57:10.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:09 smithi181 ceph-mon[41183]: failed to encode map e302 with expected crc 2024-03-29T04:57:10.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:09 smithi181 ceph-mon[41183]: failed to encode map e302 with expected crc 2024-03-29T04:57:10.942 INFO:tasks.workunit.client.0.smithi181.stdout:migration_abort 2024-03-29T04:57:11.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:10 smithi187 ceph-mon[33595]: failed to encode map e303 with expected crc 2024-03-29T04:57:11.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:10 smithi187 ceph-mon[33595]: osdmap e303: 8 total, 8 up, 8 in 2024-03-29T04:57:11.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:10 smithi187 ceph-mon[33595]: failed to encode map e303 with expected crc 2024-03-29T04:57:11.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:10 smithi187 ceph-mon[33595]: failed to encode map e303 with expected crc 2024-03-29T04:57:11.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:10 smithi187 ceph-mon[33595]: failed to encode map e303 with expected crc 2024-03-29T04:57:11.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:10 smithi187 ceph-mon[33595]: failed to encode map e303 with expected crc 2024-03-29T04:57:11.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:10.735+0000 7f1d52566700 -1 mon.b@1(peon).osd e304 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:11.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:10 smithi181 ceph-mon[101970]: failed to encode map e303 with expected crc 2024-03-29T04:57:11.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:10 smithi181 ceph-mon[101970]: osdmap e303: 8 total, 8 up, 8 in 2024-03-29T04:57:11.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:10 smithi181 ceph-mon[101970]: failed to encode map e303 with expected crc 2024-03-29T04:57:11.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:10 smithi181 ceph-mon[101970]: failed to encode map e303 with expected crc 2024-03-29T04:57:11.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:10 smithi181 ceph-mon[101970]: failed to encode map e303 with expected crc 2024-03-29T04:57:11.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:10 smithi181 ceph-mon[101970]: failed to encode map e303 with expected crc 2024-03-29T04:57:11.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:10 smithi181 ceph-mon[41183]: failed to encode map e303 with expected crc 2024-03-29T04:57:11.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:10 smithi181 ceph-mon[41183]: osdmap e303: 8 total, 8 up, 8 in 2024-03-29T04:57:11.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:10 smithi181 ceph-mon[41183]: failed to encode map e303 with expected crc 2024-03-29T04:57:11.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:10 smithi181 ceph-mon[41183]: failed to encode map e303 with expected crc 2024-03-29T04:57:11.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:10 smithi181 ceph-mon[41183]: failed to encode map e303 with expected crc 2024-03-29T04:57:11.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:10 smithi181 ceph-mon[41183]: failed to encode map e303 with expected crc 2024-03-29T04:57:11.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:10 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:10.733+0000 7fced2a08700 -1 mon.c@2(peon).osd e304 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:11.872 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixParentReattach (4026 ms) 2024-03-29T04:57:11.872 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.CloneV2ParentAbortRelinkNotNeeded 2024-03-29T04:57:11.904 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:57:11.915 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 0~10 2024-03-29T04:57:11.923 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:57:12.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:11 smithi187 ceph-mon[33595]: pgmap v93: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 378 KiB/s rd, 25 KiB/s wr, 479 op/s 2024-03-29T04:57:12.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:11 smithi187 ceph-mon[33595]: osdmap e304: 8 total, 8 up, 8 in 2024-03-29T04:57:12.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:11 smithi187 ceph-mon[33595]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:11 smithi187 ceph-mon[33595]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:11 smithi187 ceph-mon[33595]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:11 smithi187 ceph-mon[33595]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:11 smithi187 ceph-mon[33595]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:11 smithi187 ceph-mon[33595]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:11 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:12.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:11.740+0000 7f1d52566700 -1 mon.b@1(peon).osd e305 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:12.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[101970]: pgmap v93: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 378 KiB/s rd, 25 KiB/s wr, 479 op/s 2024-03-29T04:57:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[101970]: osdmap e304: 8 total, 8 up, 8 in 2024-03-29T04:57:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[101970]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[101970]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[101970]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[101970]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[101970]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[101970]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:12.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[41183]: pgmap v93: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 378 KiB/s rd, 25 KiB/s wr, 479 op/s 2024-03-29T04:57:12.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[41183]: osdmap e304: 8 total, 8 up, 8 in 2024-03-29T04:57:12.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[41183]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[41183]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[41183]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[41183]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[41183]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[41183]: failed to encode map e304 with expected crc 2024-03-29T04:57:12.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:12.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:11.740+0000 7fced2a08700 -1 mon.c@2(peon).osd e305 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:12.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:12.437+0000 7fced2a08700 -1 mon.c@2(peon).osd e306 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:12.751 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:12 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:12.436+0000 7f1d52566700 -1 mon.b@1(peon).osd e306 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:12 smithi187 ceph-mon[33595]: osdmap e305: 8 total, 8 up, 8 in 2024-03-29T04:57:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:12 smithi187 ceph-mon[33595]: failed to encode map e305 with expected crc 2024-03-29T04:57:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:12 smithi187 ceph-mon[33595]: failed to encode map e305 with expected crc 2024-03-29T04:57:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:12 smithi187 ceph-mon[33595]: failed to encode map e305 with expected crc 2024-03-29T04:57:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:12 smithi187 ceph-mon[33595]: failed to encode map e305 with expected crc 2024-03-29T04:57:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:12 smithi187 ceph-mon[33595]: failed to encode map e305 with expected crc 2024-03-29T04:57:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:12 smithi187 ceph-mon[33595]: osdmap e306: 8 total, 8 up, 8 in 2024-03-29T04:57:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:12 smithi187 ceph-mon[33595]: failed to encode map e306 with expected crc 2024-03-29T04:57:13.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:12 smithi187 ceph-mon[33595]: failed to encode map e306 with expected crc 2024-03-29T04:57:13.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[101970]: osdmap e305: 8 total, 8 up, 8 in 2024-03-29T04:57:13.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[101970]: failed to encode map e305 with expected crc 2024-03-29T04:57:13.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[101970]: failed to encode map e305 with expected crc 2024-03-29T04:57:13.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[101970]: failed to encode map e305 with expected crc 2024-03-29T04:57:13.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[101970]: failed to encode map e305 with expected crc 2024-03-29T04:57:13.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[101970]: failed to encode map e305 with expected crc 2024-03-29T04:57:13.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[101970]: osdmap e306: 8 total, 8 up, 8 in 2024-03-29T04:57:13.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[101970]: failed to encode map e306 with expected crc 2024-03-29T04:57:13.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[101970]: failed to encode map e306 with expected crc 2024-03-29T04:57:13.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[41183]: osdmap e305: 8 total, 8 up, 8 in 2024-03-29T04:57:13.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[41183]: failed to encode map e305 with expected crc 2024-03-29T04:57:13.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[41183]: failed to encode map e305 with expected crc 2024-03-29T04:57:13.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[41183]: failed to encode map e305 with expected crc 2024-03-29T04:57:13.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[41183]: failed to encode map e305 with expected crc 2024-03-29T04:57:13.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[41183]: failed to encode map e305 with expected crc 2024-03-29T04:57:13.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[41183]: osdmap e306: 8 total, 8 up, 8 in 2024-03-29T04:57:13.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[41183]: failed to encode map e306 with expected crc 2024-03-29T04:57:13.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:12 smithi181 ceph-mon[41183]: failed to encode map e306 with expected crc 2024-03-29T04:57:13.464 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap1 2024-03-29T04:57:13.531 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:57:13.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:13.441+0000 7fced2a08700 -1 mon.c@2(peon).osd e307 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:13.753 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:13 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:13.441+0000 7f1d52566700 -1 mon.b@1(peon).osd e307 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:14.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:13 smithi187 ceph-mon[33595]: pgmap v96: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 291 KiB/s rd, 26 KiB/s wr, 387 op/s 2024-03-29T04:57:14.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:13 smithi187 ceph-mon[33595]: failed to encode map e306 with expected crc 2024-03-29T04:57:14.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:13 smithi187 ceph-mon[33595]: failed to encode map e306 with expected crc 2024-03-29T04:57:14.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:13 smithi187 ceph-mon[33595]: failed to encode map e306 with expected crc 2024-03-29T04:57:14.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:13 smithi187 ceph-mon[33595]: failed to encode map e306 with expected crc 2024-03-29T04:57:14.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:13 smithi187 ceph-mon[33595]: osdmap e307: 8 total, 8 up, 8 in 2024-03-29T04:57:14.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:13 smithi187 ceph-mon[33595]: failed to encode map e307 with expected crc 2024-03-29T04:57:14.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:13 smithi187 ceph-mon[33595]: failed to encode map e307 with expected crc 2024-03-29T04:57:14.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-mon[41183]: pgmap v96: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 291 KiB/s rd, 26 KiB/s wr, 387 op/s 2024-03-29T04:57:14.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-mon[41183]: failed to encode map e306 with expected crc 2024-03-29T04:57:14.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-mon[41183]: failed to encode map e306 with expected crc 2024-03-29T04:57:14.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-mon[41183]: failed to encode map e306 with expected crc 2024-03-29T04:57:14.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-mon[41183]: failed to encode map e306 with expected crc 2024-03-29T04:57:14.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-mon[41183]: osdmap e307: 8 total, 8 up, 8 in 2024-03-29T04:57:14.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-mon[41183]: failed to encode map e307 with expected crc 2024-03-29T04:57:14.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-mon[41183]: failed to encode map e307 with expected crc 2024-03-29T04:57:14.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-mon[101970]: pgmap v96: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 291 KiB/s rd, 26 KiB/s wr, 387 op/s 2024-03-29T04:57:14.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-mon[101970]: failed to encode map e306 with expected crc 2024-03-29T04:57:14.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-mon[101970]: failed to encode map e306 with expected crc 2024-03-29T04:57:14.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-mon[101970]: failed to encode map e306 with expected crc 2024-03-29T04:57:14.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-mon[101970]: failed to encode map e306 with expected crc 2024-03-29T04:57:14.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-mon[101970]: osdmap e307: 8 total, 8 up, 8 in 2024-03-29T04:57:14.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-mon[101970]: failed to encode map e307 with expected crc 2024-03-29T04:57:14.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:13 smithi181 ceph-mon[101970]: failed to encode map e307 with expected crc 2024-03-29T04:57:14.641 INFO:tasks.workunit.client.0.smithi181.stdout:migration_abort 2024-03-29T04:57:14.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:14 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:14.442+0000 7fced2a08700 -1 mon.c@2(peon).osd e308 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:14.761 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:14 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:14.442+0000 7f1d52566700 -1 mon.b@1(peon).osd e308 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:15.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:14 smithi187 ceph-mon[33595]: failed to encode map e307 with expected crc 2024-03-29T04:57:15.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:14 smithi187 ceph-mon[33595]: failed to encode map e307 with expected crc 2024-03-29T04:57:15.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:14 smithi187 ceph-mon[33595]: failed to encode map e308 with expected crc 2024-03-29T04:57:15.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:14 smithi187 ceph-mon[33595]: osdmap e308: 8 total, 8 up, 8 in 2024-03-29T04:57:15.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:14 smithi187 ceph-mon[33595]: failed to encode map e308 with expected crc 2024-03-29T04:57:15.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:14 smithi181 ceph-mon[41183]: failed to encode map e307 with expected crc 2024-03-29T04:57:15.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:14 smithi181 ceph-mon[41183]: failed to encode map e307 with expected crc 2024-03-29T04:57:15.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:14 smithi181 ceph-mon[41183]: failed to encode map e308 with expected crc 2024-03-29T04:57:15.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:14 smithi181 ceph-mon[41183]: osdmap e308: 8 total, 8 up, 8 in 2024-03-29T04:57:15.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:14 smithi181 ceph-mon[41183]: failed to encode map e308 with expected crc 2024-03-29T04:57:15.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:14 smithi181 ceph-mon[101970]: failed to encode map e307 with expected crc 2024-03-29T04:57:15.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:14 smithi181 ceph-mon[101970]: failed to encode map e307 with expected crc 2024-03-29T04:57:15.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:14 smithi181 ceph-mon[101970]: failed to encode map e308 with expected crc 2024-03-29T04:57:15.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:14 smithi181 ceph-mon[101970]: osdmap e308: 8 total, 8 up, 8 in 2024-03-29T04:57:15.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:14 smithi181 ceph-mon[101970]: failed to encode map e308 with expected crc 2024-03-29T04:57:15.582 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.CloneV2ParentAbortRelinkNotNeeded (3710 ms) 2024-03-29T04:57:15.582 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.StressNoMigrate 2024-03-29T04:57:15.613 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:57:15.625 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 21455610~4485589 2024-03-29T04:57:15.668 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 26376349~5861015 2024-03-29T04:57:15.670 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 77538871~6347209 2024-03-29T04:57:15.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:15.444+0000 7fced2a08700 -1 mon.c@2(peon).osd e309 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:15.720 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 28854488~4902419 2024-03-29T04:57:15.722 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 29931179~5727770 2024-03-29T04:57:15.758 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 10839548~199243 2024-03-29T04:57:15.758 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 27315310~41615 2024-03-29T04:57:15.761 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 4491141~6920294 2024-03-29T04:57:15.761 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap0 2024-03-29T04:57:15.762 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:15 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:15.444+0000 7f1d52566700 -1 mon.b@1(peon).osd e309 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:16.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:15 smithi187 ceph-mon[33595]: pgmap v99: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 291 KiB/s rd, 26 KiB/s wr, 387 op/s 2024-03-29T04:57:16.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:15 smithi187 ceph-mon[33595]: failed to encode map e308 with expected crc 2024-03-29T04:57:16.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:15 smithi187 ceph-mon[33595]: failed to encode map e308 with expected crc 2024-03-29T04:57:16.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:15 smithi187 ceph-mon[33595]: failed to encode map e309 with expected crc 2024-03-29T04:57:16.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:15 smithi187 ceph-mon[33595]: osdmap e309: 8 total, 8 up, 8 in 2024-03-29T04:57:16.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:15 smithi181 ceph-mon[101970]: pgmap v99: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 291 KiB/s rd, 26 KiB/s wr, 387 op/s 2024-03-29T04:57:16.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:15 smithi181 ceph-mon[101970]: failed to encode map e308 with expected crc 2024-03-29T04:57:16.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:15 smithi181 ceph-mon[101970]: failed to encode map e308 with expected crc 2024-03-29T04:57:16.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:15 smithi181 ceph-mon[101970]: failed to encode map e309 with expected crc 2024-03-29T04:57:16.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:15 smithi181 ceph-mon[101970]: osdmap e309: 8 total, 8 up, 8 in 2024-03-29T04:57:16.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:15 smithi181 ceph-mon[41183]: pgmap v99: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 291 KiB/s rd, 26 KiB/s wr, 387 op/s 2024-03-29T04:57:16.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:15 smithi181 ceph-mon[41183]: failed to encode map e308 with expected crc 2024-03-29T04:57:16.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:15 smithi181 ceph-mon[41183]: failed to encode map e308 with expected crc 2024-03-29T04:57:16.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:15 smithi181 ceph-mon[41183]: failed to encode map e309 with expected crc 2024-03-29T04:57:16.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:15 smithi181 ceph-mon[41183]: osdmap e309: 8 total, 8 up, 8 in 2024-03-29T04:57:17.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:16.762+0000 7f1d52566700 -1 mon.b@1(peon).osd e310 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:17.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:16 smithi187 ceph-mon[33595]: failed to encode map e309 with expected crc 2024-03-29T04:57:17.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:16 smithi187 ceph-mon[33595]: failed to encode map e309 with expected crc 2024-03-29T04:57:17.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:16 smithi181 ceph-mon[101970]: failed to encode map e309 with expected crc 2024-03-29T04:57:17.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:16 smithi181 ceph-mon[101970]: failed to encode map e309 with expected crc 2024-03-29T04:57:17.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:16 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:16.762+0000 7fced2a08700 -1 mon.c@2(peon).osd e310 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:17.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:16 smithi181 ceph-mon[41183]: failed to encode map e309 with expected crc 2024-03-29T04:57:17.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:16 smithi181 ceph-mon[41183]: failed to encode map e309 with expected crc 2024-03-29T04:57:17.798 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap0 2024-03-29T04:57:17.812 INFO:tasks.workunit.client.0.smithi181.stdout:clone image159 -> image162 2024-03-29T04:57:17.913 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 134245737 2024-03-29T04:57:17.927 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 28813177~1324587 2024-03-29T04:57:17.980 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 34794579~2749201 2024-03-29T04:57:18.002 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 53815371~5798905 2024-03-29T04:57:18.040 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 109080010~4906956 2024-03-29T04:57:18.040 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 49373554~505435 2024-03-29T04:57:18.047 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 5975557~1132066 2024-03-29T04:57:18.052 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 72611225~3527495 2024-03-29T04:57:18.092 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 63150177~3672036 2024-03-29T04:57:18.097 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:57:18.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:17 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:17.763+0000 7f1d52566700 -1 mon.b@1(peon).osd e311 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:18.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:17 smithi187 ceph-mon[33595]: pgmap v102: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:57:18.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:17 smithi187 ceph-mon[33595]: failed to encode map e310 with expected crc 2024-03-29T04:57:18.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:17 smithi187 ceph-mon[33595]: failed to encode map e310 with expected crc 2024-03-29T04:57:18.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:17 smithi187 ceph-mon[33595]: osdmap e310: 8 total, 8 up, 8 in 2024-03-29T04:57:18.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:17 smithi187 ceph-mon[33595]: failed to encode map e310 with expected crc 2024-03-29T04:57:18.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:17 smithi187 ceph-mon[33595]: failed to encode map e310 with expected crc 2024-03-29T04:57:18.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:17 smithi187 ceph-mon[33595]: failed to encode map e310 with expected crc 2024-03-29T04:57:18.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:17 smithi181 ceph-mon[101970]: pgmap v102: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:57:18.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:17 smithi181 ceph-mon[101970]: failed to encode map e310 with expected crc 2024-03-29T04:57:18.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:17 smithi181 ceph-mon[101970]: failed to encode map e310 with expected crc 2024-03-29T04:57:18.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:17 smithi181 ceph-mon[101970]: osdmap e310: 8 total, 8 up, 8 in 2024-03-29T04:57:18.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:17 smithi181 ceph-mon[101970]: failed to encode map e310 with expected crc 2024-03-29T04:57:18.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:17 smithi181 ceph-mon[101970]: failed to encode map e310 with expected crc 2024-03-29T04:57:18.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:17 smithi181 ceph-mon[101970]: failed to encode map e310 with expected crc 2024-03-29T04:57:18.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:17 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:17.765+0000 7fced2a08700 -1 mon.c@2(peon).osd e311 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:18.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:17 smithi181 ceph-mon[41183]: pgmap v102: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:57:18.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:17 smithi181 ceph-mon[41183]: failed to encode map e310 with expected crc 2024-03-29T04:57:18.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:17 smithi181 ceph-mon[41183]: failed to encode map e310 with expected crc 2024-03-29T04:57:18.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:17 smithi181 ceph-mon[41183]: osdmap e310: 8 total, 8 up, 8 in 2024-03-29T04:57:18.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:17 smithi181 ceph-mon[41183]: failed to encode map e310 with expected crc 2024-03-29T04:57:18.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:17 smithi181 ceph-mon[41183]: failed to encode map e310 with expected crc 2024-03-29T04:57:18.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:17 smithi181 ceph-mon[41183]: failed to encode map e310 with expected crc 2024-03-29T04:57:19.133 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:57:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:57:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:57:19.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:18.778+0000 7f1d52566700 -1 mon.b@1(peon).osd e312 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:19.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:18 smithi187 ceph-mon[33595]: osdmap e311: 8 total, 8 up, 8 in 2024-03-29T04:57:19.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:18 smithi187 ceph-mon[33595]: failed to encode map e311 with expected crc 2024-03-29T04:57:19.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:18 smithi187 ceph-mon[33595]: failed to encode map e311 with expected crc 2024-03-29T04:57:19.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:18 smithi187 ceph-mon[33595]: failed to encode map e311 with expected crc 2024-03-29T04:57:19.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:18 smithi187 ceph-mon[33595]: failed to encode map e311 with expected crc 2024-03-29T04:57:19.203 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:57:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:57:18] "GET /metrics HTTP/1.1" 200 38725 "" "Prometheus/2.43.0" 2024-03-29T04:57:19.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:18 smithi181 ceph-mon[101970]: osdmap e311: 8 total, 8 up, 8 in 2024-03-29T04:57:19.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:18 smithi181 ceph-mon[101970]: failed to encode map e311 with expected crc 2024-03-29T04:57:19.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:18 smithi181 ceph-mon[101970]: failed to encode map e311 with expected crc 2024-03-29T04:57:19.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:18 smithi181 ceph-mon[101970]: failed to encode map e311 with expected crc 2024-03-29T04:57:19.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:18 smithi181 ceph-mon[101970]: failed to encode map e311 with expected crc 2024-03-29T04:57:19.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:18.778+0000 7fced2a08700 -1 mon.c@2(peon).osd e312 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:19.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:18 smithi181 ceph-mon[41183]: osdmap e311: 8 total, 8 up, 8 in 2024-03-29T04:57:19.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:18 smithi181 ceph-mon[41183]: failed to encode map e311 with expected crc 2024-03-29T04:57:19.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:18 smithi181 ceph-mon[41183]: failed to encode map e311 with expected crc 2024-03-29T04:57:19.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:18 smithi181 ceph-mon[41183]: failed to encode map e311 with expected crc 2024-03-29T04:57:19.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:18 smithi181 ceph-mon[41183]: failed to encode map e311 with expected crc 2024-03-29T04:57:19.810 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap1 2024-03-29T04:57:19.822 INFO:tasks.workunit.client.0.smithi181.stdout:clone image162 -> image164 2024-03-29T04:57:19.954 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 185661162 2024-03-29T04:57:19.965 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 178203436~7457726 2024-03-29T04:57:20.013 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 66377225~6017187 2024-03-29T04:57:20.047 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 74885558~8151234 2024-03-29T04:57:20.109 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 116340930~1417847 2024-03-29T04:57:20.115 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 180878622~2161335 2024-03-29T04:57:20.129 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 67074637~6890892 2024-03-29T04:57:20.132 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 93620638~2100978 2024-03-29T04:57:20.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:19 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:19.779+0000 7f1d52566700 -1 mon.b@1(peon).osd e313 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:20.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:19 smithi187 ceph-mon[33595]: pgmap v105: 137 pgs: 137 active+clean; 944 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 259 KiB/s rd, 7.9 MiB/s wr, 345 op/s 2024-03-29T04:57:20.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:19 smithi187 ceph-mon[33595]: osdmap e312: 8 total, 8 up, 8 in 2024-03-29T04:57:20.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:19 smithi187 ceph-mon[33595]: failed to encode map e312 with expected crc 2024-03-29T04:57:20.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:19 smithi187 ceph-mon[33595]: failed to encode map e312 with expected crc 2024-03-29T04:57:20.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:19 smithi187 ceph-mon[33595]: failed to encode map e312 with expected crc 2024-03-29T04:57:20.148 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 27281609~4938546 2024-03-29T04:57:20.178 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap2 2024-03-29T04:57:20.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:19 smithi181 ceph-mon[101970]: pgmap v105: 137 pgs: 137 active+clean; 944 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 259 KiB/s rd, 7.9 MiB/s wr, 345 op/s 2024-03-29T04:57:20.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:19 smithi181 ceph-mon[101970]: osdmap e312: 8 total, 8 up, 8 in 2024-03-29T04:57:20.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:19 smithi181 ceph-mon[101970]: failed to encode map e312 with expected crc 2024-03-29T04:57:20.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:19 smithi181 ceph-mon[101970]: failed to encode map e312 with expected crc 2024-03-29T04:57:20.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:19 smithi181 ceph-mon[101970]: failed to encode map e312 with expected crc 2024-03-29T04:57:20.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:19 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:19.780+0000 7fced2a08700 -1 mon.c@2(peon).osd e313 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:20.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:19 smithi181 ceph-mon[41183]: pgmap v105: 137 pgs: 137 active+clean; 944 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 259 KiB/s rd, 7.9 MiB/s wr, 345 op/s 2024-03-29T04:57:20.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:19 smithi181 ceph-mon[41183]: osdmap e312: 8 total, 8 up, 8 in 2024-03-29T04:57:20.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:19 smithi181 ceph-mon[41183]: failed to encode map e312 with expected crc 2024-03-29T04:57:20.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:19 smithi181 ceph-mon[41183]: failed to encode map e312 with expected crc 2024-03-29T04:57:20.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:19 smithi181 ceph-mon[41183]: failed to encode map e312 with expected crc 2024-03-29T04:57:21.016 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade resume' 2024-03-29T04:57:21.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:20 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:20.792+0000 7f1d52566700 -1 mon.b@1(peon).osd e314 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:21.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:20 smithi187 ceph-mon[33595]: osdmap e313: 8 total, 8 up, 8 in 2024-03-29T04:57:21.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:20 smithi187 ceph-mon[33595]: failed to encode map e313 with expected crc 2024-03-29T04:57:21.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:20 smithi187 ceph-mon[33595]: failed to encode map e313 with expected crc 2024-03-29T04:57:21.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:20 smithi187 ceph-mon[33595]: failed to encode map e313 with expected crc 2024-03-29T04:57:21.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:20 smithi187 ceph-mon[33595]: failed to encode map e313 with expected crc 2024-03-29T04:57:21.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:20 smithi181 ceph-mon[101970]: osdmap e313: 8 total, 8 up, 8 in 2024-03-29T04:57:21.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:20 smithi181 ceph-mon[101970]: failed to encode map e313 with expected crc 2024-03-29T04:57:21.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:20 smithi181 ceph-mon[101970]: failed to encode map e313 with expected crc 2024-03-29T04:57:21.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:20 smithi181 ceph-mon[101970]: failed to encode map e313 with expected crc 2024-03-29T04:57:21.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:20 smithi181 ceph-mon[101970]: failed to encode map e313 with expected crc 2024-03-29T04:57:21.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:20.793+0000 7fced2a08700 -1 mon.c@2(peon).osd e314 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:21.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:20 smithi181 ceph-mon[41183]: osdmap e313: 8 total, 8 up, 8 in 2024-03-29T04:57:21.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:20 smithi181 ceph-mon[41183]: failed to encode map e313 with expected crc 2024-03-29T04:57:21.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:20 smithi181 ceph-mon[41183]: failed to encode map e313 with expected crc 2024-03-29T04:57:21.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:20 smithi181 ceph-mon[41183]: failed to encode map e313 with expected crc 2024-03-29T04:57:21.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:20 smithi181 ceph-mon[41183]: failed to encode map e313 with expected crc 2024-03-29T04:57:21.275 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:21.828 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap2 2024-03-29T04:57:21.838 INFO:tasks.workunit.client.0.smithi181.stdout:clone image164 -> image166 2024-03-29T04:57:21.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:21 smithi181 ceph-mon[101970]: pgmap v108: 137 pgs: 137 active+clean; 944 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 259 KiB/s rd, 7.9 MiB/s wr, 345 op/s 2024-03-29T04:57:21.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:21 smithi181 ceph-mon[101970]: osdmap e314: 8 total, 8 up, 8 in 2024-03-29T04:57:21.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:21 smithi181 ceph-mon[101970]: failed to encode map e314 with expected crc 2024-03-29T04:57:21.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:21 smithi181 ceph-mon[101970]: failed to encode map e314 with expected crc 2024-03-29T04:57:21.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:21 smithi181 ceph-mon[101970]: failed to encode map e314 with expected crc 2024-03-29T04:57:21.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:21 smithi181 ceph-mon[101970]: failed to encode map e314 with expected crc 2024-03-29T04:57:21.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:21 smithi181 ceph-mon[101970]: failed to encode map e314 with expected crc 2024-03-29T04:57:21.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:21.794+0000 7fced2a08700 -1 mon.c@2(peon).osd e315 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:21.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:21 smithi181 ceph-mon[41183]: pgmap v108: 137 pgs: 137 active+clean; 944 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 259 KiB/s rd, 7.9 MiB/s wr, 345 op/s 2024-03-29T04:57:21.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:21 smithi181 ceph-mon[41183]: osdmap e314: 8 total, 8 up, 8 in 2024-03-29T04:57:21.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:21 smithi181 ceph-mon[41183]: failed to encode map e314 with expected crc 2024-03-29T04:57:21.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:21 smithi181 ceph-mon[41183]: failed to encode map e314 with expected crc 2024-03-29T04:57:21.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:21 smithi181 ceph-mon[41183]: failed to encode map e314 with expected crc 2024-03-29T04:57:21.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:21 smithi181 ceph-mon[41183]: failed to encode map e314 with expected crc 2024-03-29T04:57:21.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:21 smithi181 ceph-mon[41183]: failed to encode map e314 with expected crc 2024-03-29T04:57:21.980 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 161353043 2024-03-29T04:57:21.998 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 107929609~8312757 2024-03-29T04:57:22.051 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 25273836~7682113 2024-03-29T04:57:22.066 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 50234954~7046826 2024-03-29T04:57:22.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:21 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:21.794+0000 7f1d52566700 -1 mon.b@1(peon).osd e315 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:22.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:21 smithi187 ceph-mon[33595]: pgmap v108: 137 pgs: 137 active+clean; 944 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 259 KiB/s rd, 7.9 MiB/s wr, 345 op/s 2024-03-29T04:57:22.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:21 smithi187 ceph-mon[33595]: osdmap e314: 8 total, 8 up, 8 in 2024-03-29T04:57:22.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:21 smithi187 ceph-mon[33595]: failed to encode map e314 with expected crc 2024-03-29T04:57:22.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:21 smithi187 ceph-mon[33595]: failed to encode map e314 with expected crc 2024-03-29T04:57:22.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:21 smithi187 ceph-mon[33595]: failed to encode map e314 with expected crc 2024-03-29T04:57:22.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:21 smithi187 ceph-mon[33595]: failed to encode map e314 with expected crc 2024-03-29T04:57:22.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:21 smithi187 ceph-mon[33595]: failed to encode map e314 with expected crc 2024-03-29T04:57:22.134 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 96136874~6866675 2024-03-29T04:57:22.161 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 151272582~7559020 2024-03-29T04:57:22.204 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 54255315~2302778 2024-03-29T04:57:22.207 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 76643868~1316247 2024-03-29T04:57:22.257 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 150631350~6439431 2024-03-29T04:57:22.261 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap3 2024-03-29T04:57:22.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:22.438+0000 7fced2a08700 -1 mon.c@2(peon).osd e316 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:22.816 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:22.437+0000 7f1d52566700 -1 mon.b@1(peon).osd e316 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:23.017 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[101970]: osdmap e315: 8 total, 8 up, 8 in 2024-03-29T04:57:23.017 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[101970]: failed to encode map e315 with expected crc 2024-03-29T04:57:23.017 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[101970]: failed to encode map e315 with expected crc 2024-03-29T04:57:23.018 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[101970]: failed to encode map e315 with expected crc 2024-03-29T04:57:23.018 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[101970]: failed to encode map e315 with expected crc 2024-03-29T04:57:23.018 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[101970]: failed to encode map e315 with expected crc 2024-03-29T04:57:23.018 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:57:23.018 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[101970]: osdmap e316: 8 total, 8 up, 8 in 2024-03-29T04:57:23.018 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[101970]: failed to encode map e316 with expected crc 2024-03-29T04:57:23.018 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[41183]: osdmap e315: 8 total, 8 up, 8 in 2024-03-29T04:57:23.018 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[41183]: failed to encode map e315 with expected crc 2024-03-29T04:57:23.018 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[41183]: failed to encode map e315 with expected crc 2024-03-29T04:57:23.018 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[41183]: failed to encode map e315 with expected crc 2024-03-29T04:57:23.018 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[41183]: failed to encode map e315 with expected crc 2024-03-29T04:57:23.019 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[41183]: failed to encode map e315 with expected crc 2024-03-29T04:57:23.019 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:57:23.019 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[41183]: osdmap e316: 8 total, 8 up, 8 in 2024-03-29T04:57:23.019 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:22 smithi181 ceph-mon[41183]: failed to encode map e316 with expected crc 2024-03-29T04:57:23.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:22 smithi187 ceph-mon[33595]: osdmap e315: 8 total, 8 up, 8 in 2024-03-29T04:57:23.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:22 smithi187 ceph-mon[33595]: failed to encode map e315 with expected crc 2024-03-29T04:57:23.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:22 smithi187 ceph-mon[33595]: failed to encode map e315 with expected crc 2024-03-29T04:57:23.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:22 smithi187 ceph-mon[33595]: failed to encode map e315 with expected crc 2024-03-29T04:57:23.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:22 smithi187 ceph-mon[33595]: failed to encode map e315 with expected crc 2024-03-29T04:57:23.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:22 smithi187 ceph-mon[33595]: failed to encode map e315 with expected crc 2024-03-29T04:57:23.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:22 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:57:23.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:22 smithi187 ceph-mon[33595]: osdmap e316: 8 total, 8 up, 8 in 2024-03-29T04:57:23.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:22 smithi187 ceph-mon[33595]: failed to encode map e316 with expected crc 2024-03-29T04:57:23.462 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap3 2024-03-29T04:57:23.472 INFO:tasks.workunit.client.0.smithi181.stdout:clone image166 -> image168 2024-03-29T04:57:23.494 INFO:teuthology.orchestra.run.smithi181.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:57:23.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:23 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:23.440+0000 7f1d52566700 -1 mon.b@1(peon).osd e317 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:23.645 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 110106157 2024-03-29T04:57:23.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:23 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:23.440+0000 7fced2a08700 -1 mon.c@2(peon).osd e317 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:23.842 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.StressNoMigrate (8260 ms) 2024-03-29T04:57:23.842 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.Stress 2024-03-29T04:57:23.877 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:57:23.888 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 48214001~6060011 2024-03-29T04:57:23.933 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 4562295~2778847 2024-03-29T04:57:23.933 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 38434257~71698 2024-03-29T04:57:23.938 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 72119108~7020031 2024-03-29T04:57:23.938 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 23665281~224468 2024-03-29T04:57:23.944 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 25641891~5352005 2024-03-29T04:57:23.944 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 77073866~6812214 2024-03-29T04:57:23.988 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 58200077~8158180 2024-03-29T04:57:23.989 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap0 2024-03-29T04:57:24.093 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:23 smithi187 ceph-mon[33595]: pgmap v111: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 15 MiB/s rd, 32 MiB/s wr, 677 op/s 2024-03-29T04:57:24.093 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:23 smithi187 ceph-mon[33595]: failed to encode map e316 with expected crc 2024-03-29T04:57:24.093 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:23 smithi187 ceph-mon[33595]: failed to encode map e316 with expected crc 2024-03-29T04:57:24.093 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:23 smithi187 ceph-mon[33595]: failed to encode map e316 with expected crc 2024-03-29T04:57:24.093 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:23 smithi187 ceph-mon[33595]: failed to encode map e316 with expected crc 2024-03-29T04:57:24.093 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:23 smithi187 ceph-mon[33595]: osdmap e317: 8 total, 8 up, 8 in 2024-03-29T04:57:24.093 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:23 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:24.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:23 smithi181 ceph-mon[41183]: pgmap v111: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 15 MiB/s rd, 32 MiB/s wr, 677 op/s 2024-03-29T04:57:24.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:23 smithi181 ceph-mon[41183]: failed to encode map e316 with expected crc 2024-03-29T04:57:24.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:23 smithi181 ceph-mon[41183]: failed to encode map e316 with expected crc 2024-03-29T04:57:24.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:23 smithi181 ceph-mon[41183]: failed to encode map e316 with expected crc 2024-03-29T04:57:24.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:23 smithi181 ceph-mon[41183]: failed to encode map e316 with expected crc 2024-03-29T04:57:24.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:23 smithi181 ceph-mon[41183]: osdmap e317: 8 total, 8 up, 8 in 2024-03-29T04:57:24.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:23 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:24.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:23 smithi181 ceph-mon[101970]: pgmap v111: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 15 MiB/s rd, 32 MiB/s wr, 677 op/s 2024-03-29T04:57:24.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:23 smithi181 ceph-mon[101970]: failed to encode map e316 with expected crc 2024-03-29T04:57:24.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:23 smithi181 ceph-mon[101970]: failed to encode map e316 with expected crc 2024-03-29T04:57:24.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:23 smithi181 ceph-mon[101970]: failed to encode map e316 with expected crc 2024-03-29T04:57:24.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:23 smithi181 ceph-mon[101970]: failed to encode map e316 with expected crc 2024-03-29T04:57:24.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:23 smithi181 ceph-mon[101970]: osdmap e317: 8 total, 8 up, 8 in 2024-03-29T04:57:24.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:23 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:24.848 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'echo wait for majority of mons to upgrade' 2024-03-29T04:57:25.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:24 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:24.816+0000 7f1d52566700 -1 mon.b@1(peon).osd e318 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:25.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:24 smithi187 ceph-mon[33595]: failed to encode map e317 with expected crc 2024-03-29T04:57:25.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:24 smithi187 ceph-mon[33595]: failed to encode map e317 with expected crc 2024-03-29T04:57:25.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:24 smithi187 ceph-mon[33595]: failed to encode map e317 with expected crc 2024-03-29T04:57:25.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:24 smithi187 ceph-mon[33595]: failed to encode map e317 with expected crc 2024-03-29T04:57:25.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:24 smithi187 ceph-mon[33595]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:57:25.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:24 smithi187 ceph-mon[33595]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:57:25.143 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:24 smithi181 ceph-mon[101970]: failed to encode map e317 with expected crc 2024-03-29T04:57:25.143 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:24 smithi181 ceph-mon[101970]: failed to encode map e317 with expected crc 2024-03-29T04:57:25.143 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:24 smithi181 ceph-mon[101970]: failed to encode map e317 with expected crc 2024-03-29T04:57:25.143 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:24 smithi181 ceph-mon[101970]: failed to encode map e317 with expected crc 2024-03-29T04:57:25.143 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:24 smithi181 ceph-mon[101970]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:57:25.143 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:24 smithi181 ceph-mon[101970]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:57:25.144 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:24 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:24.814+0000 7fced2a08700 -1 mon.c@2(peon).osd e318 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:25.144 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:24 smithi181 ceph-mon[41183]: failed to encode map e317 with expected crc 2024-03-29T04:57:25.144 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:24 smithi181 ceph-mon[41183]: failed to encode map e317 with expected crc 2024-03-29T04:57:25.144 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:24 smithi181 ceph-mon[41183]: failed to encode map e317 with expected crc 2024-03-29T04:57:25.144 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:24 smithi181 ceph-mon[41183]: failed to encode map e317 with expected crc 2024-03-29T04:57:25.144 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:24 smithi181 ceph-mon[41183]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:57:25.144 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:24 smithi181 ceph-mon[41183]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:57:25.776 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:25.844 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 7187674~6346497 2024-03-29T04:57:25.855 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 25370842~1760984 2024-03-29T04:57:25.855 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 44126268~1101844 2024-03-29T04:57:25.855 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 17493524~1521266 2024-03-29T04:57:25.855 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 10494483~234051 2024-03-29T04:57:25.886 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 4343677~93399 2024-03-29T04:57:25.886 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 39707842~4108331 2024-03-29T04:57:25.887 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 15275786~2226030 2024-03-29T04:57:25.888 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:57:25.902 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:25.816+0000 7fced2a08700 -1 mon.c@2(peon).osd e319 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:25.902 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:25 smithi181 ceph-mon[41183]: pgmap v114: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 15 MiB/s rd, 32 MiB/s wr, 677 op/s 2024-03-29T04:57:25.902 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:25 smithi181 ceph-mon[41183]: osdmap e318: 8 total, 8 up, 8 in 2024-03-29T04:57:25.902 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:25 smithi181 ceph-mon[41183]: failed to encode map e318 with expected crc 2024-03-29T04:57:25.903 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:25 smithi181 ceph-mon[41183]: failed to encode map e318 with expected crc 2024-03-29T04:57:25.903 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:25 smithi181 ceph-mon[41183]: failed to encode map e318 with expected crc 2024-03-29T04:57:25.903 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:25 smithi181 ceph-mon[41183]: failed to encode map e318 with expected crc 2024-03-29T04:57:25.903 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:25 smithi181 ceph-mon[101970]: pgmap v114: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 15 MiB/s rd, 32 MiB/s wr, 677 op/s 2024-03-29T04:57:25.903 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:25 smithi181 ceph-mon[101970]: osdmap e318: 8 total, 8 up, 8 in 2024-03-29T04:57:25.903 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:25 smithi181 ceph-mon[101970]: failed to encode map e318 with expected crc 2024-03-29T04:57:25.903 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:25 smithi181 ceph-mon[101970]: failed to encode map e318 with expected crc 2024-03-29T04:57:25.903 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:25 smithi181 ceph-mon[101970]: failed to encode map e318 with expected crc 2024-03-29T04:57:25.903 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:25 smithi181 ceph-mon[101970]: failed to encode map e318 with expected crc 2024-03-29T04:57:26.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:25.816+0000 7f1d52566700 -1 mon.b@1(peon).osd e319 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:26.186 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:25 smithi187 ceph-mon[33595]: pgmap v114: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 15 MiB/s rd, 32 MiB/s wr, 677 op/s 2024-03-29T04:57:26.186 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:25 smithi187 ceph-mon[33595]: osdmap e318: 8 total, 8 up, 8 in 2024-03-29T04:57:26.186 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:25 smithi187 ceph-mon[33595]: failed to encode map e318 with expected crc 2024-03-29T04:57:26.186 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:25 smithi187 ceph-mon[33595]: failed to encode map e318 with expected crc 2024-03-29T04:57:26.186 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:25 smithi187 ceph-mon[33595]: failed to encode map e318 with expected crc 2024-03-29T04:57:26.187 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:25 smithi187 ceph-mon[33595]: failed to encode map e318 with expected crc 2024-03-29T04:57:26.838 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:26.827+0000 7fced2a08700 -1 mon.c@2(peon).osd e320 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:26.839 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[101970]: failed to encode map e319 with expected crc 2024-03-29T04:57:26.839 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[101970]: failed to encode map e319 with expected crc 2024-03-29T04:57:26.839 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[101970]: osdmap e319: 8 total, 8 up, 8 in 2024-03-29T04:57:26.839 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[101970]: failed to encode map e319 with expected crc 2024-03-29T04:57:26.839 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[101970]: failed to encode map e319 with expected crc 2024-03-29T04:57:26.839 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[101970]: failed to encode map e319 with expected crc 2024-03-29T04:57:26.839 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[101970]: failed to encode map e319 with expected crc 2024-03-29T04:57:26.839 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:26.839 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:26.839 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:27.107 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[41183]: failed to encode map e319 with expected crc 2024-03-29T04:57:27.108 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[41183]: failed to encode map e319 with expected crc 2024-03-29T04:57:27.108 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[41183]: osdmap e319: 8 total, 8 up, 8 in 2024-03-29T04:57:27.108 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[41183]: failed to encode map e319 with expected crc 2024-03-29T04:57:27.108 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[41183]: failed to encode map e319 with expected crc 2024-03-29T04:57:27.108 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[41183]: failed to encode map e319 with expected crc 2024-03-29T04:57:27.108 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[41183]: failed to encode map e319 with expected crc 2024-03-29T04:57:27.108 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:27.108 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:27.108 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:26 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:27.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:26 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:26.827+0000 7f1d52566700 -1 mon.b@1(peon).osd e320 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:27.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:26 smithi187 ceph-mon[33595]: failed to encode map e319 with expected crc 2024-03-29T04:57:27.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:26 smithi187 ceph-mon[33595]: failed to encode map e319 with expected crc 2024-03-29T04:57:27.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:26 smithi187 ceph-mon[33595]: osdmap e319: 8 total, 8 up, 8 in 2024-03-29T04:57:27.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:26 smithi187 ceph-mon[33595]: failed to encode map e319 with expected crc 2024-03-29T04:57:27.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:26 smithi187 ceph-mon[33595]: failed to encode map e319 with expected crc 2024-03-29T04:57:27.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:26 smithi187 ceph-mon[33595]: failed to encode map e319 with expected crc 2024-03-29T04:57:27.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:26 smithi187 ceph-mon[33595]: failed to encode map e319 with expected crc 2024-03-29T04:57:27.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:26 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:27.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:26 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:27.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:26 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:27.574 INFO:teuthology.orchestra.run.smithi181.stdout:wait for majority of mons to upgrade 2024-03-29T04:57:27.857 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap1 2024-03-29T04:57:27.868 INFO:tasks.workunit.client.0.smithi181.stdout:clone image169 -> image172 2024-03-29T04:57:27.961 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 59678776~3378395 2024-03-29T04:57:27.992 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 62323119~7307972 2024-03-29T04:57:27.998 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 21048586~7846586 2024-03-29T04:57:28.059 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 39824624~4078930 2024-03-29T04:57:28.088 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 45791542~2081964 2024-03-29T04:57:28.126 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 6939384~5750014 2024-03-29T04:57:28.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:27.829+0000 7f1d52566700 -1 mon.b@1(peon).osd e321 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:28.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:27 smithi187 ceph-mon[33595]: pgmap v117: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:57:28.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:27 smithi187 ceph-mon[33595]: osdmap e320: 8 total, 8 up, 8 in 2024-03-29T04:57:28.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:27 smithi187 ceph-mon[33595]: failed to encode map e320 with expected crc 2024-03-29T04:57:28.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:27 smithi187 ceph-mon[33595]: failed to encode map e320 with expected crc 2024-03-29T04:57:28.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:27 smithi187 ceph-mon[33595]: failed to encode map e320 with expected crc 2024-03-29T04:57:28.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:27 smithi187 ceph-mon[33595]: failed to encode map e320 with expected crc 2024-03-29T04:57:28.141 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 62428159~6439074 2024-03-29T04:57:28.186 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 50143099~5738450 2024-03-29T04:57:28.190 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap2 2024-03-29T04:57:28.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:27 smithi181 ceph-mon[101970]: pgmap v117: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:57:28.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:27 smithi181 ceph-mon[101970]: osdmap e320: 8 total, 8 up, 8 in 2024-03-29T04:57:28.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:27 smithi181 ceph-mon[101970]: failed to encode map e320 with expected crc 2024-03-29T04:57:28.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:27 smithi181 ceph-mon[101970]: failed to encode map e320 with expected crc 2024-03-29T04:57:28.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:27 smithi181 ceph-mon[101970]: failed to encode map e320 with expected crc 2024-03-29T04:57:28.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:27 smithi181 ceph-mon[101970]: failed to encode map e320 with expected crc 2024-03-29T04:57:28.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:27 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:27.829+0000 7fced2a08700 -1 mon.c@2(peon).osd e321 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:28.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:27 smithi181 ceph-mon[41183]: pgmap v117: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2024-03-29T04:57:28.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:27 smithi181 ceph-mon[41183]: osdmap e320: 8 total, 8 up, 8 in 2024-03-29T04:57:28.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:27 smithi181 ceph-mon[41183]: failed to encode map e320 with expected crc 2024-03-29T04:57:28.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:27 smithi181 ceph-mon[41183]: failed to encode map e320 with expected crc 2024-03-29T04:57:28.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:27 smithi181 ceph-mon[41183]: failed to encode map e320 with expected crc 2024-03-29T04:57:28.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:27 smithi181 ceph-mon[41183]: failed to encode map e320 with expected crc 2024-03-29T04:57:28.414 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-03-29T04:57:29.050 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:28.842+0000 7f1d52566700 -1 mon.b@1(peon).osd e322 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:29.051 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:28 smithi187 ceph-mon[33595]: osdmap e321: 8 total, 8 up, 8 in 2024-03-29T04:57:29.051 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:28 smithi187 ceph-mon[33595]: failed to encode map e321 with expected crc 2024-03-29T04:57:29.051 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:28 smithi187 ceph-mon[33595]: failed to encode map e321 with expected crc 2024-03-29T04:57:29.051 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:28 smithi187 ceph-mon[33595]: failed to encode map e321 with expected crc 2024-03-29T04:57:29.051 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:28 smithi187 ceph-mon[33595]: failed to encode map e321 with expected crc 2024-03-29T04:57:29.051 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:28 smithi187 ceph-mon[33595]: failed to encode map e321 with expected crc 2024-03-29T04:57:29.051 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:57:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:57:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:57:29.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:57:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:57:28] "GET /metrics HTTP/1.1" 200 38728 "" "Prometheus/2.43.0" 2024-03-29T04:57:29.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:28 smithi181 ceph-mon[101970]: osdmap e321: 8 total, 8 up, 8 in 2024-03-29T04:57:29.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:28 smithi181 ceph-mon[101970]: failed to encode map e321 with expected crc 2024-03-29T04:57:29.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:28 smithi181 ceph-mon[101970]: failed to encode map e321 with expected crc 2024-03-29T04:57:29.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:28 smithi181 ceph-mon[101970]: failed to encode map e321 with expected crc 2024-03-29T04:57:29.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:28 smithi181 ceph-mon[101970]: failed to encode map e321 with expected crc 2024-03-29T04:57:29.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:28 smithi181 ceph-mon[101970]: failed to encode map e321 with expected crc 2024-03-29T04:57:29.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:28.842+0000 7fced2a08700 -1 mon.c@2(peon).osd e322 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:29.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:28 smithi181 ceph-mon[41183]: osdmap e321: 8 total, 8 up, 8 in 2024-03-29T04:57:29.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:28 smithi181 ceph-mon[41183]: failed to encode map e321 with expected crc 2024-03-29T04:57:29.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:28 smithi181 ceph-mon[41183]: failed to encode map e321 with expected crc 2024-03-29T04:57:29.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:28 smithi181 ceph-mon[41183]: failed to encode map e321 with expected crc 2024-03-29T04:57:29.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:28 smithi181 ceph-mon[41183]: failed to encode map e321 with expected crc 2024-03-29T04:57:29.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:28 smithi181 ceph-mon[41183]: failed to encode map e321 with expected crc 2024-03-29T04:57:29.241 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:29.880 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap2 2024-03-29T04:57:29.890 INFO:tasks.workunit.client.0.smithi181.stdout:clone image172 -> image174 2024-03-29T04:57:29.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:29 smithi181 ceph-mon[41183]: pgmap v120: 137 pgs: 137 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 19 MiB/s wr, 383 op/s 2024-03-29T04:57:29.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:29 smithi181 ceph-mon[41183]: osdmap e322: 8 total, 8 up, 8 in 2024-03-29T04:57:29.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:29 smithi181 ceph-mon[41183]: failed to encode map e322 with expected crc 2024-03-29T04:57:29.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:29 smithi181 ceph-mon[41183]: failed to encode map e322 with expected crc 2024-03-29T04:57:29.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:29 smithi181 ceph-mon[41183]: failed to encode map e322 with expected crc 2024-03-29T04:57:29.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:29 smithi181 ceph-mon[41183]: failed to encode map e322 with expected crc 2024-03-29T04:57:29.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:29 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:29.851+0000 7fced2a08700 -1 mon.c@2(peon).osd e323 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:29.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:29 smithi181 ceph-mon[101970]: pgmap v120: 137 pgs: 137 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 19 MiB/s wr, 383 op/s 2024-03-29T04:57:29.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:29 smithi181 ceph-mon[101970]: osdmap e322: 8 total, 8 up, 8 in 2024-03-29T04:57:29.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:29 smithi181 ceph-mon[101970]: failed to encode map e322 with expected crc 2024-03-29T04:57:29.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:29 smithi181 ceph-mon[101970]: failed to encode map e322 with expected crc 2024-03-29T04:57:29.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:29 smithi181 ceph-mon[101970]: failed to encode map e322 with expected crc 2024-03-29T04:57:29.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:29 smithi181 ceph-mon[101970]: failed to encode map e322 with expected crc 2024-03-29T04:57:29.997 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 94269367 2024-03-29T04:57:30.009 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 73490572~5434187 2024-03-29T04:57:30.062 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 92040636~2228731 2024-03-29T04:57:30.062 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 22584621~7650002 2024-03-29T04:57:30.126 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 76567807~3259786 2024-03-29T04:57:30.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:29 smithi187 ceph-mon[33595]: pgmap v120: 137 pgs: 137 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 19 MiB/s wr, 383 op/s 2024-03-29T04:57:30.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:29 smithi187 ceph-mon[33595]: osdmap e322: 8 total, 8 up, 8 in 2024-03-29T04:57:30.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:29 smithi187 ceph-mon[33595]: failed to encode map e322 with expected crc 2024-03-29T04:57:30.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:29 smithi187 ceph-mon[33595]: failed to encode map e322 with expected crc 2024-03-29T04:57:30.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:29 smithi187 ceph-mon[33595]: failed to encode map e322 with expected crc 2024-03-29T04:57:30.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:29 smithi187 ceph-mon[33595]: failed to encode map e322 with expected crc 2024-03-29T04:57:30.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:29 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:29.851+0000 7f1d52566700 -1 mon.b@1(peon).osd e323 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:30.171 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 17360677~409830 2024-03-29T04:57:30.179 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 65471953~7741901 2024-03-29T04:57:30.216 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 80618428~5136752 2024-03-29T04:57:30.247 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 51092547~1626922 2024-03-29T04:57:30.253 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap3 2024-03-29T04:57:30.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:30 smithi181 ceph-mon[101970]: osdmap e323: 8 total, 8 up, 8 in 2024-03-29T04:57:30.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:30 smithi181 ceph-mon[101970]: failed to encode map e323 with expected crc 2024-03-29T04:57:30.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:30 smithi181 ceph-mon[101970]: failed to encode map e323 with expected crc 2024-03-29T04:57:30.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:30 smithi181 ceph-mon[101970]: failed to encode map e323 with expected crc 2024-03-29T04:57:30.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:30 smithi181 ceph-mon[101970]: failed to encode map e323 with expected crc 2024-03-29T04:57:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:30 smithi181 ceph-mon[41183]: osdmap e323: 8 total, 8 up, 8 in 2024-03-29T04:57:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:30 smithi181 ceph-mon[41183]: failed to encode map e323 with expected crc 2024-03-29T04:57:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:30 smithi181 ceph-mon[41183]: failed to encode map e323 with expected crc 2024-03-29T04:57:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:30 smithi181 ceph-mon[41183]: failed to encode map e323 with expected crc 2024-03-29T04:57:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:30 smithi181 ceph-mon[41183]: failed to encode map e323 with expected crc 2024-03-29T04:57:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:30.859+0000 7fced2a08700 -1 mon.c@2(peon).osd e324 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:31.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:30 smithi187 ceph-mon[33595]: osdmap e323: 8 total, 8 up, 8 in 2024-03-29T04:57:31.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:30 smithi187 ceph-mon[33595]: failed to encode map e323 with expected crc 2024-03-29T04:57:31.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:30 smithi187 ceph-mon[33595]: failed to encode map e323 with expected crc 2024-03-29T04:57:31.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:30 smithi187 ceph-mon[33595]: failed to encode map e323 with expected crc 2024-03-29T04:57:31.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:30 smithi187 ceph-mon[33595]: failed to encode map e323 with expected crc 2024-03-29T04:57:31.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:30 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:30.860+0000 7f1d52566700 -1 mon.b@1(peon).osd e324 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:31.897 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:57:32.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:31.860+0000 7f1d52566700 -1 mon.b@1(peon).osd e325 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:32.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:31 smithi187 ceph-mon[33595]: pgmap v123: 137 pgs: 137 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 19 MiB/s wr, 383 op/s 2024-03-29T04:57:32.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:31 smithi187 ceph-mon[33595]: osdmap e324: 8 total, 8 up, 8 in 2024-03-29T04:57:32.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:31 smithi187 ceph-mon[33595]: failed to encode map e324 with expected crc 2024-03-29T04:57:32.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:31 smithi187 ceph-mon[33595]: failed to encode map e324 with expected crc 2024-03-29T04:57:32.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:31 smithi187 ceph-mon[33595]: failed to encode map e324 with expected crc 2024-03-29T04:57:32.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:31 smithi187 ceph-mon[33595]: failed to encode map e324 with expected crc 2024-03-29T04:57:32.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:31 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/3132279802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:57:32.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:31.860+0000 7fced2a08700 -1 mon.c@2(peon).osd e325 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:32.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:31 smithi181 ceph-mon[41183]: pgmap v123: 137 pgs: 137 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 19 MiB/s wr, 383 op/s 2024-03-29T04:57:32.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:31 smithi181 ceph-mon[41183]: osdmap e324: 8 total, 8 up, 8 in 2024-03-29T04:57:32.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:31 smithi181 ceph-mon[41183]: failed to encode map e324 with expected crc 2024-03-29T04:57:32.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:31 smithi181 ceph-mon[41183]: failed to encode map e324 with expected crc 2024-03-29T04:57:32.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:31 smithi181 ceph-mon[41183]: failed to encode map e324 with expected crc 2024-03-29T04:57:32.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:31 smithi181 ceph-mon[41183]: failed to encode map e324 with expected crc 2024-03-29T04:57:32.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:31 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/3132279802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:57:32.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:31 smithi181 ceph-mon[101970]: pgmap v123: 137 pgs: 137 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 19 MiB/s rd, 19 MiB/s wr, 383 op/s 2024-03-29T04:57:32.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:31 smithi181 ceph-mon[101970]: osdmap e324: 8 total, 8 up, 8 in 2024-03-29T04:57:32.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:31 smithi181 ceph-mon[101970]: failed to encode map e324 with expected crc 2024-03-29T04:57:32.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:31 smithi181 ceph-mon[101970]: failed to encode map e324 with expected crc 2024-03-29T04:57:32.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:31 smithi181 ceph-mon[101970]: failed to encode map e324 with expected crc 2024-03-29T04:57:32.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:31 smithi181 ceph-mon[101970]: failed to encode map e324 with expected crc 2024-03-29T04:57:32.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:31 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3132279802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:57:33.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:32 smithi187 ceph-mon[33595]: osdmap e325: 8 total, 8 up, 8 in 2024-03-29T04:57:33.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:32 smithi187 ceph-mon[33595]: failed to encode map e325 with expected crc 2024-03-29T04:57:33.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:32 smithi187 ceph-mon[33595]: failed to encode map e325 with expected crc 2024-03-29T04:57:33.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:32 smithi187 ceph-mon[33595]: failed to encode map e325 with expected crc 2024-03-29T04:57:33.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:32 smithi187 ceph-mon[33595]: failed to encode map e325 with expected crc 2024-03-29T04:57:33.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:32 smithi187 ceph-mon[33595]: failed to encode map e325 with expected crc 2024-03-29T04:57:33.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:32 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:33.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:32 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:33.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:32 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:57:33.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:32 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:57:33.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:32 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:33.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:32 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:57:33.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:32 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:57:33.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:32 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:57:33.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:32 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-03-29T04:57:33.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:32 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:32.977+0000 7f1d52566700 -1 mon.b@1(peon).osd e326 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:33.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[101970]: osdmap e325: 8 total, 8 up, 8 in 2024-03-29T04:57:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[101970]: failed to encode map e325 with expected crc 2024-03-29T04:57:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[101970]: failed to encode map e325 with expected crc 2024-03-29T04:57:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[101970]: failed to encode map e325 with expected crc 2024-03-29T04:57:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[101970]: failed to encode map e325 with expected crc 2024-03-29T04:57:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[101970]: failed to encode map e325 with expected crc 2024-03-29T04:57:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:57:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:57:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:57:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:57:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:57:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-03-29T04:57:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[41183]: osdmap e325: 8 total, 8 up, 8 in 2024-03-29T04:57:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[41183]: failed to encode map e325 with expected crc 2024-03-29T04:57:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[41183]: failed to encode map e325 with expected crc 2024-03-29T04:57:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[41183]: failed to encode map e325 with expected crc 2024-03-29T04:57:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[41183]: failed to encode map e325 with expected crc 2024-03-29T04:57:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[41183]: failed to encode map e325 with expected crc 2024-03-29T04:57:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:57:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:57:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:57:33.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:57:33.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:57:33.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-03-29T04:57:33.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:32 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:32.977+0000 7fced2a08700 -1 mon.c@2(peon).osd e326 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:33.405 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:57:33.607 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:57:34.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:33 smithi187 ceph-mon[33595]: pgmap v126: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 12 MiB/s rd, 25 MiB/s wr, 561 op/s 2024-03-29T04:57:34.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:33 smithi187 ceph-mon[33595]: pgmap v127: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 13 MiB/s rd, 27 MiB/s wr, 607 op/s 2024-03-29T04:57:34.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:33 smithi187 ceph-mon[33595]: Upgrade: It appears safe to stop mon.c 2024-03-29T04:57:34.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:33 smithi187 ceph-mon[33595]: osdmap e326: 8 total, 8 up, 8 in 2024-03-29T04:57:34.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:33 smithi187 ceph-mon[33595]: failed to encode map e326 with expected crc 2024-03-29T04:57:34.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:33 smithi187 ceph-mon[33595]: failed to encode map e326 with expected crc 2024-03-29T04:57:34.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:33 smithi187 ceph-mon[33595]: failed to encode map e326 with expected crc 2024-03-29T04:57:34.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:33 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:33.978+0000 7f1d52566700 -1 mon.b@1(peon).osd e327 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:34.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:33 smithi181 ceph-mon[101970]: pgmap v126: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 12 MiB/s rd, 25 MiB/s wr, 561 op/s 2024-03-29T04:57:34.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:33 smithi181 ceph-mon[101970]: pgmap v127: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 13 MiB/s rd, 27 MiB/s wr, 607 op/s 2024-03-29T04:57:34.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:33 smithi181 ceph-mon[101970]: Upgrade: It appears safe to stop mon.c 2024-03-29T04:57:34.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:33 smithi181 ceph-mon[101970]: osdmap e326: 8 total, 8 up, 8 in 2024-03-29T04:57:34.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:33 smithi181 ceph-mon[101970]: failed to encode map e326 with expected crc 2024-03-29T04:57:34.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:33 smithi181 ceph-mon[101970]: failed to encode map e326 with expected crc 2024-03-29T04:57:34.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:33 smithi181 ceph-mon[101970]: failed to encode map e326 with expected crc 2024-03-29T04:57:34.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:33 smithi181 ceph-mon[41183]: pgmap v126: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 12 MiB/s rd, 25 MiB/s wr, 561 op/s 2024-03-29T04:57:34.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:33 smithi181 ceph-mon[41183]: pgmap v127: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 13 MiB/s rd, 27 MiB/s wr, 607 op/s 2024-03-29T04:57:34.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:33 smithi181 ceph-mon[41183]: Upgrade: It appears safe to stop mon.c 2024-03-29T04:57:34.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:33 smithi181 ceph-mon[41183]: osdmap e326: 8 total, 8 up, 8 in 2024-03-29T04:57:34.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:33 smithi181 ceph-mon[41183]: failed to encode map e326 with expected crc 2024-03-29T04:57:34.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:33 smithi181 ceph-mon[41183]: failed to encode map e326 with expected crc 2024-03-29T04:57:34.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:33 smithi181 ceph-mon[41183]: failed to encode map e326 with expected crc 2024-03-29T04:57:34.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:33 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:33.978+0000 7fced2a08700 -1 mon.c@2(peon).osd e327 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:34.248 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.Stress (10405 ms) 2024-03-29T04:57:34.248 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.Stress2 2024-03-29T04:57:34.279 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:57:34.289 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 47983024~4429938 2024-03-29T04:57:34.324 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 8278440~7980256 2024-03-29T04:57:34.324 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 48474072~3401390 2024-03-29T04:57:34.344 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 74218157~6328907 2024-03-29T04:57:34.344 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 31306182~3880875 2024-03-29T04:57:34.372 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 44275602~2633607 2024-03-29T04:57:34.374 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 66788050~8135037 2024-03-29T04:57:34.433 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 60695416~1500557 2024-03-29T04:57:34.433 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap0 2024-03-29T04:57:35.291 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[41183]: osdmap e327: 8 total, 8 up, 8 in 2024-03-29T04:57:35.291 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[41183]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.291 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[41183]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.291 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[41183]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.291 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[41183]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.291 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[41183]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.291 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[41183]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.291 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[41183]: from='client.? 172.21.15.181:0/2539070719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:57:35.291 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[41183]: Upgrade: Updating mon.c 2024-03-29T04:57:35.292 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[41183]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:35.292 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:57:35.292 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:57:35.292 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[41183]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:57:35.292 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[41183]: Deploying daemon mon.c on smithi181 2024-03-29T04:57:35.292 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[41183]: pgmap v130: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 13 MiB/s rd, 27 MiB/s wr, 610 op/s 2024-03-29T04:57:35.292 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[101970]: osdmap e327: 8 total, 8 up, 8 in 2024-03-29T04:57:35.292 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[101970]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.292 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[101970]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.292 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[101970]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.293 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[101970]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.293 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[101970]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.293 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[101970]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.293 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2539070719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:57:35.293 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[101970]: Upgrade: Updating mon.c 2024-03-29T04:57:35.293 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[101970]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:35.293 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:57:35.293 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:57:35.293 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:57:35.293 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[101970]: Deploying daemon mon.c on smithi181 2024-03-29T04:57:35.293 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:34 smithi181 ceph-mon[101970]: pgmap v130: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 13 MiB/s rd, 27 MiB/s wr, 610 op/s 2024-03-29T04:57:35.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:34 smithi187 ceph-mon[33595]: osdmap e327: 8 total, 8 up, 8 in 2024-03-29T04:57:35.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:34 smithi187 ceph-mon[33595]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:34 smithi187 ceph-mon[33595]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:34 smithi187 ceph-mon[33595]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:34 smithi187 ceph-mon[33595]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:34 smithi187 ceph-mon[33595]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:34 smithi187 ceph-mon[33595]: failed to encode map e327 with expected crc 2024-03-29T04:57:35.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:34 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/2539070719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:57:35.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:34 smithi187 ceph-mon[33595]: Upgrade: Updating mon.c 2024-03-29T04:57:35.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:34 smithi187 ceph-mon[33595]: from='mgr.25337 ' entity='mgr.y' 2024-03-29T04:57:35.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:34 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:57:35.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:34 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:57:35.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:34 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:57:35.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:34 smithi187 ceph-mon[33595]: Deploying daemon mon.c on smithi181 2024-03-29T04:57:35.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:34 smithi187 ceph-mon[33595]: pgmap v130: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 13 MiB/s rd, 27 MiB/s wr, 610 op/s 2024-03-29T04:57:35.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:35 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:35.432+0000 7fced2a08700 -1 mon.c@2(peon).osd e328 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:35 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:35.432+0000 7f1d52566700 -1 mon.b@1(peon).osd e328 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:36.473 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap0 2024-03-29T04:57:36.487 INFO:tasks.workunit.client.0.smithi181.stdout:clone image175 -> image178 2024-03-29T04:57:36.583 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 77587899~158960 2024-03-29T04:57:36.594 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 76066262~6176756 2024-03-29T04:57:36.599 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 80037297~3848783 2024-03-29T04:57:36.632 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 50928575~3276348 2024-03-29T04:57:36.647 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 4903271~5841864 2024-03-29T04:57:36.654 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-mon[101970]: osdmap e328: 8 total, 8 up, 8 in 2024-03-29T04:57:36.655 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-mon[101970]: failed to encode map e328 with expected crc 2024-03-29T04:57:36.655 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-mon[101970]: failed to encode map e328 with expected crc 2024-03-29T04:57:36.655 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-mon[101970]: failed to encode map e328 with expected crc 2024-03-29T04:57:36.655 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-mon[101970]: failed to encode map e328 with expected crc 2024-03-29T04:57:36.655 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:36 smithi181 systemd[1]: Stopping Ceph mon.c for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:57:36.655 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:36.435+0000 7fced2a08700 -1 mon.c@2(peon).osd e329 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:36.655 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-mon[41183]: osdmap e328: 8 total, 8 up, 8 in 2024-03-29T04:57:36.655 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-mon[41183]: failed to encode map e328 with expected crc 2024-03-29T04:57:36.656 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-mon[41183]: failed to encode map e328 with expected crc 2024-03-29T04:57:36.656 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-mon[41183]: failed to encode map e328 with expected crc 2024-03-29T04:57:36.656 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-mon[41183]: failed to encode map e328 with expected crc 2024-03-29T04:57:36.656 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:36.647+0000 7fced8213700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:57:36.656 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[41160]: 2024-03-29T04:57:36.647+0000 7fced8213700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-03-29T04:57:36.689 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 14000264~947935 2024-03-29T04:57:36.694 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 22155089~2412955 2024-03-29T04:57:36.711 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 35559036~3263032 2024-03-29T04:57:36.732 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:57:36.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:36 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:36.436+0000 7f1d52566700 -1 mon.b@1(peon).osd e329 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:36.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:36 smithi187 ceph-mon[33595]: osdmap e328: 8 total, 8 up, 8 in 2024-03-29T04:57:36.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:36 smithi187 ceph-mon[33595]: failed to encode map e328 with expected crc 2024-03-29T04:57:36.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:36 smithi187 ceph-mon[33595]: failed to encode map e328 with expected crc 2024-03-29T04:57:36.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:36 smithi187 ceph-mon[33595]: failed to encode map e328 with expected crc 2024-03-29T04:57:36.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:36 smithi187 ceph-mon[33595]: failed to encode map e328 with expected crc 2024-03-29T04:57:36.909 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:57:36] ENGINE Bus STOPPING 2024-03-29T04:57:36.909 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:57:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:57:36.909 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:57:36] ENGINE Bus STOPPED 2024-03-29T04:57:36.909 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:57:36] ENGINE Bus STARTING 2024-03-29T04:57:36.909 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:57:36] ENGINE Serving on http://:::9283 2024-03-29T04:57:36.909 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:57:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:04:57:36] ENGINE Bus STARTED 2024-03-29T04:57:37.192 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:36 smithi181 podman[106256]: 2024-03-29 04:57:36.902590309 +0000 UTC m=+0.432256628 container died 16379529f880470e0d1be0f5e1a7793dca0e4d97691cd19fce2c20f3043fca9e (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=quincy-4213e17, org.label-schema.schema-version=1.0, ceph=True) 2024-03-29T04:57:37.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:37 smithi181 podman[106256]: 2024-03-29 04:57:37.40247115 +0000 UTC m=+0.932137469 container cleanup 16379529f880470e0d1be0f5e1a7793dca0e4d97691cd19fce2c20f3043fca9e (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326) 2024-03-29T04:57:37.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:37 smithi181 bash[106256]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c 2024-03-29T04:57:37.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:37 smithi181 podman[106289]: 2024-03-29 04:57:37.612486387 +0000 UTC m=+0.707787744 container remove 16379529f880470e0d1be0f5e1a7793dca0e4d97691cd19fce2c20f3043fca9e (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240326) 2024-03-29T04:57:37.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:37 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.c.service: Deactivated successfully. 2024-03-29T04:57:37.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:37 smithi181 systemd[1]: Stopped Ceph mon.c for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:57:37.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:37 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.c.service: Consumed 8.964s CPU time. 2024-03-29T04:57:38.349 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:37 smithi181 systemd[1]: Starting Ceph mon.c for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:57:38.349 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 podman[106429]: 2024-03-29 04:57:38.068995555 +0000 UTC m=+0.018229339 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:57:38.349 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 podman[106429]: 2024-03-29 04:57:38.188689822 +0000 UTC m=+0.137923635 container create 10f02d533230221001aa56d0e2d12679381982e09fd7b5fa62741baad2c211d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:57:38.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 podman[106429]: 2024-03-29 04:57:38.46699629 +0000 UTC m=+0.416230078 container init 10f02d533230221001aa56d0e2d12679381982e09fd7b5fa62741baad2c211d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, ceph=True) 2024-03-29T04:57:38.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 podman[106429]: 2024-03-29 04:57:38.472789205 +0000 UTC m=+0.422022976 container start 10f02d533230221001aa56d0e2d12679381982e09fd7b5fa62741baad2c211d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-03-29T04:57:38.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: set uid:gid to 167:167 (ceph:ceph) 2024-03-29T04:57:38.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev), process ceph-mon, pid 7 2024-03-29T04:57:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: pidfile_write: ignore empty --pid-file 2024-03-29T04:57:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: load: jerasure load: lrc 2024-03-29T04:57:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: RocksDB version: 7.9.2 2024-03-29T04:57:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Git sha 0 2024-03-29T04:57:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Compile date 2024-03-28 15:35:57 2024-03-29T04:57:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: DB SUMMARY 2024-03-29T04:57:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: DB Session ID: SS0ULQIIPZHL45UZMT6L 2024-03-29T04:57:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: CURRENT file: CURRENT 2024-03-29T04:57:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: IDENTITY file: IDENTITY 2024-03-29T04:57:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1099 Bytes 2024-03-29T04:57:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-03-29T04:57:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 9907779 ; 2024-03-29T04:57:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.error_if_exists: 0 2024-03-29T04:57:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.create_if_missing: 0 2024-03-29T04:57:38.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.paranoid_checks: 1 2024-03-29T04:57:38.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-29T04:57:38.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-29T04:57:38.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-29T04:57:38.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.env: 0x562897860c60 2024-03-29T04:57:38.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.fs: PosixFileSystem 2024-03-29T04:57:38.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.info_log: 0x562898c1dae0 2024-03-29T04:57:38.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_file_opening_threads: 16 2024-03-29T04:57:38.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.statistics: (nil) 2024-03-29T04:57:38.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.use_fsync: 0 2024-03-29T04:57:38.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_log_file_size: 0 2024-03-29T04:57:38.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-29T04:57:38.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-29T04:57:38.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.keep_log_file_num: 1000 2024-03-29T04:57:38.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.recycle_log_file_num: 0 2024-03-29T04:57:38.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.allow_fallocate: 1 2024-03-29T04:57:38.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.allow_mmap_reads: 0 2024-03-29T04:57:38.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.allow_mmap_writes: 0 2024-03-29T04:57:38.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.use_direct_reads: 0 2024-03-29T04:57:38.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-29T04:57:38.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.create_missing_column_families: 0 2024-03-29T04:57:38.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.db_log_dir: 2024-03-29T04:57:38.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.wal_dir: 2024-03-29T04:57:38.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-29T04:57:38.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-29T04:57:38.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-29T04:57:38.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-29T04:57:38.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-29T04:57:38.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-29T04:57:38.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.advise_random_on_open: 1 2024-03-29T04:57:38.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.db_write_buffer_size: 0 2024-03-29T04:57:38.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.write_buffer_manager: 0x562898bf0820 2024-03-29T04:57:38.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-29T04:57:38.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-29T04:57:38.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-29T04:57:38.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.rate_limiter: (nil) 2024-03-29T04:57:38.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-29T04:57:38.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.wal_recovery_mode: 2 2024-03-29T04:57:38.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.enable_thread_tracking: 0 2024-03-29T04:57:38.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.enable_pipelined_write: 0 2024-03-29T04:57:38.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.unordered_write: 0 2024-03-29T04:57:38.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-29T04:57:38.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-29T04:57:38.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-29T04:57:38.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-29T04:57:38.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.row_cache: None 2024-03-29T04:57:38.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.wal_filter: None 2024-03-29T04:57:38.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-29T04:57:38.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.allow_ingest_behind: 0 2024-03-29T04:57:38.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.two_write_queues: 0 2024-03-29T04:57:38.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.manual_wal_flush: 0 2024-03-29T04:57:38.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.wal_compression: 0 2024-03-29T04:57:38.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.atomic_flush: 0 2024-03-29T04:57:38.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-29T04:57:38.711 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-29T04:57:38.711 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-29T04:57:38.711 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.log_readahead_size: 0 2024-03-29T04:57:38.711 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-29T04:57:38.711 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.best_efforts_recovery: 0 2024-03-29T04:57:38.711 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-29T04:57:38.711 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-29T04:57:38.711 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.allow_data_in_errors: 0 2024-03-29T04:57:38.711 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.db_host_id: __hostname__ 2024-03-29T04:57:38.711 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-29T04:57:38.711 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_background_jobs: 2 2024-03-29T04:57:38.711 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_background_compactions: -1 2024-03-29T04:57:38.711 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_subcompactions: 1 2024-03-29T04:57:38.711 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-29T04:57:38.711 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-29T04:57:38.712 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-29T04:57:38.712 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_total_wal_size: 0 2024-03-29T04:57:38.712 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-29T04:57:38.712 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-29T04:57:38.712 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-29T04:57:38.712 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-29T04:57:38.712 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_open_files: -1 2024-03-29T04:57:38.712 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.bytes_per_sync: 0 2024-03-29T04:57:38.712 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-29T04:57:38.712 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-29T04:57:38.712 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compaction_readahead_size: 0 2024-03-29T04:57:38.712 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_background_flushes: -1 2024-03-29T04:57:38.712 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Compression algorithms supported: 2024-03-29T04:57:38.712 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-29T04:57:38.712 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: kZSTD supported: 0 2024-03-29T04:57:38.713 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: kXpressCompression supported: 0 2024-03-29T04:57:38.713 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: kLZ4HCCompression supported: 1 2024-03-29T04:57:38.713 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: kZlibCompression supported: 1 2024-03-29T04:57:38.713 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: kSnappyCompression supported: 1 2024-03-29T04:57:38.713 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: kLZ4Compression supported: 1 2024-03-29T04:57:38.713 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: kBZip2Compression supported: 0 2024-03-29T04:57:38.713 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-29T04:57:38.713 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-29T04:57:38.713 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-03-29T04:57:38.713 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-29T04:57:38.713 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-29T04:57:38.713 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.merge_operator: 2024-03-29T04:57:38.713 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compaction_filter: None 2024-03-29T04:57:38.713 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compaction_filter_factory: None 2024-03-29T04:57:38.714 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.sst_partitioner_factory: None 2024-03-29T04:57:38.714 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-29T04:57:38.714 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-29T04:57:38.714 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562898c1dbe0) 2024-03-29T04:57:38.714 INFO:journalctl@ceph.mon.c.smithi181.stdout: cache_index_and_filter_blocks: 1 2024-03-29T04:57:38.714 INFO:journalctl@ceph.mon.c.smithi181.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-29T04:57:38.714 INFO:journalctl@ceph.mon.c.smithi181.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-29T04:57:38.714 INFO:journalctl@ceph.mon.c.smithi181.stdout: pin_top_level_index_and_filter: 1 2024-03-29T04:57:38.714 INFO:journalctl@ceph.mon.c.smithi181.stdout: index_type: 0 2024-03-29T04:57:38.714 INFO:journalctl@ceph.mon.c.smithi181.stdout: data_block_index_type: 0 2024-03-29T04:57:38.715 INFO:journalctl@ceph.mon.c.smithi181.stdout: index_shortening: 1 2024-03-29T04:57:38.715 INFO:journalctl@ceph.mon.c.smithi181.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-29T04:57:38.715 INFO:journalctl@ceph.mon.c.smithi181.stdout: checksum: 4 2024-03-29T04:57:38.715 INFO:journalctl@ceph.mon.c.smithi181.stdout: no_block_cache: 0 2024-03-29T04:57:38.715 INFO:journalctl@ceph.mon.c.smithi181.stdout: block_cache: 0x562898c21090 2024-03-29T04:57:38.715 INFO:journalctl@ceph.mon.c.smithi181.stdout: block_cache_name: BinnedLRUCache 2024-03-29T04:57:38.715 INFO:journalctl@ceph.mon.c.smithi181.stdout: block_cache_options: 2024-03-29T04:57:38.715 INFO:journalctl@ceph.mon.c.smithi181.stdout: capacity : 536870912 2024-03-29T04:57:38.715 INFO:journalctl@ceph.mon.c.smithi181.stdout: num_shard_bits : 4 2024-03-29T04:57:38.715 INFO:journalctl@ceph.mon.c.smithi181.stdout: strict_capacity_limit : 0 2024-03-29T04:57:38.715 INFO:journalctl@ceph.mon.c.smithi181.stdout: high_pri_pool_ratio: 0.000 2024-03-29T04:57:38.716 INFO:journalctl@ceph.mon.c.smithi181.stdout: block_cache_compressed: (nil) 2024-03-29T04:57:38.716 INFO:journalctl@ceph.mon.c.smithi181.stdout: persistent_cache: (nil) 2024-03-29T04:57:38.716 INFO:journalctl@ceph.mon.c.smithi181.stdout: block_size: 4096 2024-03-29T04:57:38.716 INFO:journalctl@ceph.mon.c.smithi181.stdout: block_size_deviation: 10 2024-03-29T04:57:38.716 INFO:journalctl@ceph.mon.c.smithi181.stdout: block_restart_interval: 16 2024-03-29T04:57:38.716 INFO:journalctl@ceph.mon.c.smithi181.stdout: index_block_restart_interval: 1 2024-03-29T04:57:38.716 INFO:journalctl@ceph.mon.c.smithi181.stdout: metadata_block_size: 4096 2024-03-29T04:57:38.716 INFO:journalctl@ceph.mon.c.smithi181.stdout: partition_filters: 0 2024-03-29T04:57:38.716 INFO:journalctl@ceph.mon.c.smithi181.stdout: use_delta_encoding: 1 2024-03-29T04:57:38.716 INFO:journalctl@ceph.mon.c.smithi181.stdout: filter_policy: bloomfilter 2024-03-29T04:57:38.716 INFO:journalctl@ceph.mon.c.smithi181.stdout: whole_key_filtering: 1 2024-03-29T04:57:38.716 INFO:journalctl@ceph.mon.c.smithi181.stdout: verify_compression: 0 2024-03-29T04:57:38.716 INFO:journalctl@ceph.mon.c.smithi181.stdout: read_amp_bytes_per_bit: 0 2024-03-29T04:57:38.716 INFO:journalctl@ceph.mon.c.smithi181.stdout: format_version: 5 2024-03-29T04:57:38.717 INFO:journalctl@ceph.mon.c.smithi181.stdout: enable_index_compression: 1 2024-03-29T04:57:38.717 INFO:journalctl@ceph.mon.c.smithi181.stdout: block_align: 0 2024-03-29T04:57:38.717 INFO:journalctl@ceph.mon.c.smithi181.stdout: max_auto_readahead_size: 262144 2024-03-29T04:57:38.717 INFO:journalctl@ceph.mon.c.smithi181.stdout: prepopulate_block_cache: 0 2024-03-29T04:57:38.717 INFO:journalctl@ceph.mon.c.smithi181.stdout: initial_auto_readahead_size: 8192 2024-03-29T04:57:38.717 INFO:journalctl@ceph.mon.c.smithi181.stdout: num_file_reads_for_auto_readahead: 2 2024-03-29T04:57:38.717 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.write_buffer_size: 33554432 2024-03-29T04:57:38.717 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_write_buffer_number: 2 2024-03-29T04:57:38.717 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compression: NoCompression 2024-03-29T04:57:38.717 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.bottommost_compression: Disabled 2024-03-29T04:57:38.717 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.prefix_extractor: nullptr 2024-03-29T04:57:38.717 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-29T04:57:38.717 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.num_levels: 7 2024-03-29T04:57:38.717 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-29T04:57:38.717 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-29T04:57:38.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-29T04:57:38.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-29T04:57:38.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-29T04:57:38.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-29T04:57:38.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-29T04:57:38.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:57:38.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-29T04:57:38.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-29T04:57:38.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:57:38.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:57:38.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-29T04:57:38.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compression_opts.level: 32767 2024-03-29T04:57:38.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compression_opts.strategy: 0 2024-03-29T04:57:38.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-29T04:57:38.719 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:57:38.719 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:57:38.719 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-29T04:57:38.719 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compression_opts.enabled: false 2024-03-29T04:57:38.719 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:57:38.719 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-29T04:57:38.719 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-29T04:57:38.719 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-29T04:57:38.719 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.target_file_size_base: 67108864 2024-03-29T04:57:38.720 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-29T04:57:38.720 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-29T04:57:38.720 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-29T04:57:38.720 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-29T04:57:38.720 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-29T04:57:38.720 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-29T04:57:38.720 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-29T04:57:38.720 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-29T04:57:38.720 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-29T04:57:38.720 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-29T04:57:38.720 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-29T04:57:38.721 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-29T04:57:38.721 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-29T04:57:38.721 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-29T04:57:38.721 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.arena_block_size: 1048576 2024-03-29T04:57:38.721 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-29T04:57:38.721 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-29T04:57:38.721 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.disable_auto_compactions: 0 2024-03-29T04:57:38.721 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-29T04:57:38.721 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-29T04:57:38.721 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-29T04:57:38.721 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-29T04:57:38.721 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-29T04:57:38.721 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-29T04:57:38.721 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-29T04:57:38.721 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-29T04:57:38.722 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-29T04:57:38.722 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-29T04:57:38.722 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.table_properties_collectors: 2024-03-29T04:57:38.722 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.inplace_update_support: 0 2024-03-29T04:57:38.722 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-29T04:57:38.722 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-29T04:57:38.722 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-29T04:57:38.722 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-29T04:57:38.722 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.bloom_locality: 0 2024-03-29T04:57:38.722 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.max_successive_merges: 0 2024-03-29T04:57:38.722 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-29T04:57:38.722 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.paranoid_file_checks: 0 2024-03-29T04:57:38.722 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.force_consistency_checks: 1 2024-03-29T04:57:38.722 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.report_bg_io_stats: 0 2024-03-29T04:57:38.723 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.ttl: 2592000 2024-03-29T04:57:38.723 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-29T04:57:38.723 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-29T04:57:38.723 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-29T04:57:38.723 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.enable_blob_files: false 2024-03-29T04:57:38.723 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.min_blob_size: 0 2024-03-29T04:57:38.723 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.blob_file_size: 268435456 2024-03-29T04:57:38.723 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-29T04:57:38.723 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-29T04:57:38.723 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-29T04:57:38.723 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-29T04:57:38.723 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-29T04:57:38.723 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.blob_file_starting_level: 0 2024-03-29T04:57:38.723 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-29T04:57:38.723 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-03-29T04:57:38.724 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:57:38.724 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13426, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-29T04:57:38.724 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-03-29T04:57:38.724 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 43b1cc4c-c43f-45ef-8c9b-08bd59041353 2024-03-29T04:57:38.724 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711688258503211, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-03-29T04:57:38.724 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-03-29T04:57:38.724 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711688258527666, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 6278447, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13431, "largest_seqno": 17612, "table_properties": {"data_size": 6259565, "index_size": 12933, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4933, "raw_key_size": 45741, "raw_average_key_size": 23, "raw_value_size": 6219142, "raw_average_value_size": 3161, "num_data_blocks": 554, "num_entries": 1967, "num_filter_entries": 1967, "num_deletions": 257, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1711688258, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "43b1cc4c-c43f-45ef-8c9b-08bd59041353", "db_session_id": "SS0ULQIIPZHL45UZMT6L", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-03-29T04:57:38.724 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711688258527749, "job": 1, "event": "recovery_finished"} 2024-03-29T04:57:38.724 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-03-29T04:57:38.724 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:57:38.724 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-29T04:57:38.724 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x562898d12000 2024-03-29T04:57:38.724 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: DB pointer 0x562898cfc000 2024-03-29T04:57:38.724 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-29T04:57:38.724 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-29T04:57:38.725 INFO:journalctl@ceph.mon.c.smithi181.stdout: ** DB Stats ** 2024-03-29T04:57:38.725 INFO:journalctl@ceph.mon.c.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:57:38.725 INFO:journalctl@ceph.mon.c.smithi181.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-29T04:57:38.725 INFO:journalctl@ceph.mon.c.smithi181.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:57:38.725 INFO:journalctl@ceph.mon.c.smithi181.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:57:38.725 INFO:journalctl@ceph.mon.c.smithi181.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-29T04:57:38.725 INFO:journalctl@ceph.mon.c.smithi181.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:57:38.725 INFO:journalctl@ceph.mon.c.smithi181.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:57:38.725 INFO:journalctl@ceph.mon.c.smithi181.stdout: 2024-03-29T04:57:38.725 INFO:journalctl@ceph.mon.c.smithi181.stdout: ** Compaction Stats [default] ** 2024-03-29T04:57:38.725 INFO:journalctl@ceph.mon.c.smithi181.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-29T04:57:38.725 INFO:journalctl@ceph.mon.c.smithi181.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-29T04:57:38.725 INFO:journalctl@ceph.mon.c.smithi181.stdout: L0 1/0 5.99 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 570.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-03-29T04:57:38.725 INFO:journalctl@ceph.mon.c.smithi181.stdout: L6 1/0 8.35 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-29T04:57:38.725 INFO:journalctl@ceph.mon.c.smithi181.stdout: Sum 2/0 14.34 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 570.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-03-29T04:57:38.726 INFO:journalctl@ceph.mon.c.smithi181.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 570.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-03-29T04:57:38.726 INFO:journalctl@ceph.mon.c.smithi181.stdout: 2024-03-29T04:57:38.726 INFO:journalctl@ceph.mon.c.smithi181.stdout: ** Compaction Stats [default] ** 2024-03-29T04:57:38.726 INFO:journalctl@ceph.mon.c.smithi181.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-29T04:57:38.726 INFO:journalctl@ceph.mon.c.smithi181.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:57:38.726 INFO:journalctl@ceph.mon.c.smithi181.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 570.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-03-29T04:57:38.726 INFO:journalctl@ceph.mon.c.smithi181.stdout: 2024-03-29T04:57:38.726 INFO:journalctl@ceph.mon.c.smithi181.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-29T04:57:38.726 INFO:journalctl@ceph.mon.c.smithi181.stdout: 2024-03-29T04:57:38.726 INFO:journalctl@ceph.mon.c.smithi181.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:57:38.726 INFO:journalctl@ceph.mon.c.smithi181.stdout: Flush(GB): cumulative 0.006, interval 0.006 2024-03-29T04:57:38.726 INFO:journalctl@ceph.mon.c.smithi181.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:57:38.726 INFO:journalctl@ceph.mon.c.smithi181.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:57:38.726 INFO:journalctl@ceph.mon.c.smithi181.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:57:38.727 INFO:journalctl@ceph.mon.c.smithi181.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:57:38.727 INFO:journalctl@ceph.mon.c.smithi181.stdout: Cumulative compaction: 0.01 GB write, 218.82 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:57:38.727 INFO:journalctl@ceph.mon.c.smithi181.stdout: Interval compaction: 0.01 GB write, 218.82 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:57:38.727 INFO:journalctl@ceph.mon.c.smithi181.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:57:38.727 INFO:journalctl@ceph.mon.c.smithi181.stdout: Block cache BinnedLRUCache@0x562898c21090#7 capacity: 512.00 MB usage: 18.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-03-29T04:57:38.727 INFO:journalctl@ceph.mon.c.smithi181.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,5.03 KB,0.000959635%) IndexBlock(1,13.08 KB,0.00249445%) Misc(1,0.00 KB,0%) 2024-03-29T04:57:38.727 INFO:journalctl@ceph.mon.c.smithi181.stdout: 2024-03-29T04:57:38.727 INFO:journalctl@ceph.mon.c.smithi181.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:57:38.727 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: starting mon.c rank 2 at public addrs [v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0] at bind addrs [v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:57:38.727 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: mon.c@-1(???) e3 preinit fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:57:38.727 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: mon.c@-1(???).mds e9 new map 2024-03-29T04:57:38.727 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: mon.c@-1(???).mds e9 print_map 2024-03-29T04:57:38.727 INFO:journalctl@ceph.mon.c.smithi181.stdout: e9 2024-03-29T04:57:38.727 INFO:journalctl@ceph.mon.c.smithi181.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-29T04:57:38.727 INFO:journalctl@ceph.mon.c.smithi181.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:57:38.728 INFO:journalctl@ceph.mon.c.smithi181.stdout: legacy client fscid: 1 2024-03-29T04:57:38.728 INFO:journalctl@ceph.mon.c.smithi181.stdout: 2024-03-29T04:57:38.728 INFO:journalctl@ceph.mon.c.smithi181.stdout: Filesystem 'foo' (1) 2024-03-29T04:57:38.728 INFO:journalctl@ceph.mon.c.smithi181.stdout: fs_name foo 2024-03-29T04:57:38.728 INFO:journalctl@ceph.mon.c.smithi181.stdout: epoch 8 2024-03-29T04:57:38.728 INFO:journalctl@ceph.mon.c.smithi181.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-29T04:57:38.728 INFO:journalctl@ceph.mon.c.smithi181.stdout: created 2024-03-29T04:51:50.886956+0000 2024-03-29T04:57:38.728 INFO:journalctl@ceph.mon.c.smithi181.stdout: modified 2024-03-29T04:52:07.243441+0000 2024-03-29T04:57:38.728 INFO:journalctl@ceph.mon.c.smithi181.stdout: tableserver 0 2024-03-29T04:57:38.728 INFO:journalctl@ceph.mon.c.smithi181.stdout: root 0 2024-03-29T04:57:38.728 INFO:journalctl@ceph.mon.c.smithi181.stdout: session_timeout 60 2024-03-29T04:57:38.728 INFO:journalctl@ceph.mon.c.smithi181.stdout: session_autoclose 300 2024-03-29T04:57:38.728 INFO:journalctl@ceph.mon.c.smithi181.stdout: max_file_size 1099511627776 2024-03-29T04:57:38.728 INFO:journalctl@ceph.mon.c.smithi181.stdout: max_xattr_size 65536 2024-03-29T04:57:38.729 INFO:journalctl@ceph.mon.c.smithi181.stdout: required_client_features {} 2024-03-29T04:57:38.729 INFO:journalctl@ceph.mon.c.smithi181.stdout: last_failure 0 2024-03-29T04:57:38.729 INFO:journalctl@ceph.mon.c.smithi181.stdout: last_failure_osd_epoch 0 2024-03-29T04:57:38.729 INFO:journalctl@ceph.mon.c.smithi181.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:57:38.729 INFO:journalctl@ceph.mon.c.smithi181.stdout: max_mds 1 2024-03-29T04:57:38.729 INFO:journalctl@ceph.mon.c.smithi181.stdout: in 0 2024-03-29T04:57:38.729 INFO:journalctl@ceph.mon.c.smithi181.stdout: up {0=24560} 2024-03-29T04:57:38.729 INFO:journalctl@ceph.mon.c.smithi181.stdout: failed 2024-03-29T04:57:38.729 INFO:journalctl@ceph.mon.c.smithi181.stdout: damaged 2024-03-29T04:57:38.729 INFO:journalctl@ceph.mon.c.smithi181.stdout: stopped 2024-03-29T04:57:38.729 INFO:journalctl@ceph.mon.c.smithi181.stdout: data_pools [4] 2024-03-29T04:57:38.729 INFO:journalctl@ceph.mon.c.smithi181.stdout: metadata_pool 3 2024-03-29T04:57:38.729 INFO:journalctl@ceph.mon.c.smithi181.stdout: inline_data disabled 2024-03-29T04:57:38.730 INFO:journalctl@ceph.mon.c.smithi181.stdout: balancer 2024-03-29T04:57:38.730 INFO:journalctl@ceph.mon.c.smithi181.stdout: bal_rank_mask -1 2024-03-29T04:57:38.730 INFO:journalctl@ceph.mon.c.smithi181.stdout: standby_count_wanted 1 2024-03-29T04:57:38.730 INFO:journalctl@ceph.mon.c.smithi181.stdout: qdb_cluster leader: 0 members: 2024-03-29T04:57:38.730 INFO:journalctl@ceph.mon.c.smithi181.stdout: [mds.foo.smithi181.pflrwm{0:24560} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.181:6834/2778467308,v1:172.21.15.181:6835/2778467308] compat {c=[1],r=[1],i=[7ff]}] 2024-03-29T04:57:38.730 INFO:journalctl@ceph.mon.c.smithi181.stdout: 2024-03-29T04:57:38.730 INFO:journalctl@ceph.mon.c.smithi181.stdout: 2024-03-29T04:57:38.730 INFO:journalctl@ceph.mon.c.smithi181.stdout: Standby daemons: 2024-03-29T04:57:38.730 INFO:journalctl@ceph.mon.c.smithi181.stdout: 2024-03-29T04:57:38.730 INFO:journalctl@ceph.mon.c.smithi181.stdout: [mds.foo.smithi187.vcsnqx{-1:24511} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.187:6832/1383464008,v1:172.21.15.187:6833/1383464008] compat {c=[1],r=[1],i=[7ff]}] 2024-03-29T04:57:38.730 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: mon.c@-1(???).osd e329 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-29T04:57:38.731 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: mon.c@-1(???).osd e329 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:57:38.731 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: mon.c@-1(???).osd e329 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:57:38.731 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: mon.c@-1(???).osd e329 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:57:38.731 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-mon[106466]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-03-29T04:57:38.731 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 bash[106429]: 10f02d533230221001aa56d0e2d12679381982e09fd7b5fa62741baad2c211d2 2024-03-29T04:57:38.731 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:38 smithi181 systemd[1]: Started Ceph mon.c for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:57:39.133 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:57:38 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:57:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:57:39.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:57:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:57:38] "GET /metrics HTTP/1.1" 200 38728 "" "Prometheus/2.43.0" 2024-03-29T04:57:44.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: mon.c calling monitor election 2024-03-29T04:57:44.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: mon.b calling monitor election 2024-03-29T04:57:44.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: mon.a calling monitor election 2024-03-29T04:57:44.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:57:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: monmap epoch 3 2024-03-29T04:57:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:57:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: last_changed 2024-03-29T04:46:30.517496+0000 2024-03-29T04:57:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: created 2024-03-29T04:44:46.123593+0000 2024-03-29T04:57:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: min_mon_release 17 (quincy) 2024-03-29T04:57:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: election_strategy: 1 2024-03-29T04:57:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: 0: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.a 2024-03-29T04:57:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: 1: [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon.b 2024-03-29T04:57:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: 2: [v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0] mon.c 2024-03-29T04:57:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:57:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: osdmap e329: 8 total, 8 up, 8 in 2024-03-29T04:57:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: mgrmap e36: y(active, since 93s), standbys: x 2024-03-29T04:57:44.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-29T04:57:44.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-03-29T04:57:44.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:57:44.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: daemon grafana.a on smithi187 is in error state 2024-03-29T04:57:44.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-29T04:57:44.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: application not enabled on pool 'test-librbd-smithi181-86707-9' 2024-03-29T04:57:44.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[106466]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-29T04:57:44.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: mon.c calling monitor election 2024-03-29T04:57:44.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: mon.b calling monitor election 2024-03-29T04:57:44.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: mon.a calling monitor election 2024-03-29T04:57:44.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:57:44.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: monmap epoch 3 2024-03-29T04:57:44.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:57:44.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: last_changed 2024-03-29T04:46:30.517496+0000 2024-03-29T04:57:44.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: created 2024-03-29T04:44:46.123593+0000 2024-03-29T04:57:44.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: min_mon_release 17 (quincy) 2024-03-29T04:57:44.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: election_strategy: 1 2024-03-29T04:57:44.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: 0: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.a 2024-03-29T04:57:44.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: 1: [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon.b 2024-03-29T04:57:44.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: 2: [v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0] mon.c 2024-03-29T04:57:44.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:57:44.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: osdmap e329: 8 total, 8 up, 8 in 2024-03-29T04:57:44.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: mgrmap e36: y(active, since 93s), standbys: x 2024-03-29T04:57:44.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-29T04:57:44.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-03-29T04:57:44.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:57:44.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: daemon grafana.a on smithi187 is in error state 2024-03-29T04:57:44.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-29T04:57:44.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: application not enabled on pool 'test-librbd-smithi181-86707-9' 2024-03-29T04:57:44.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:44 smithi181 ceph-mon[101970]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-29T04:57:45.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:44.769+0000 7f1d52566700 -1 mon.b@1(peon).osd e330 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:45.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: mon.c calling monitor election 2024-03-29T04:57:45.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: mon.b calling monitor election 2024-03-29T04:57:45.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: mon.a calling monitor election 2024-03-29T04:57:45.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:57:45.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: monmap epoch 3 2024-03-29T04:57:45.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:57:45.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: last_changed 2024-03-29T04:46:30.517496+0000 2024-03-29T04:57:45.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: created 2024-03-29T04:44:46.123593+0000 2024-03-29T04:57:45.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: min_mon_release 17 (quincy) 2024-03-29T04:57:45.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: election_strategy: 1 2024-03-29T04:57:45.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: 0: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.a 2024-03-29T04:57:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: 1: [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon.b 2024-03-29T04:57:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: 2: [v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0] mon.c 2024-03-29T04:57:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:57:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: osdmap e329: 8 total, 8 up, 8 in 2024-03-29T04:57:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: mgrmap e36: y(active, since 93s), standbys: x 2024-03-29T04:57:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-29T04:57:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-03-29T04:57:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:57:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: daemon grafana.a on smithi187 is in error state 2024-03-29T04:57:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-29T04:57:45.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: application not enabled on pool 'test-librbd-smithi181-86707-9' 2024-03-29T04:57:45.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:44 smithi187 ceph-mon[33595]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-29T04:57:45.811 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap1 2024-03-29T04:57:45.823 INFO:tasks.workunit.client.0.smithi181.stdout:clone image178 -> image180 2024-03-29T04:57:45.950 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 117587540 2024-03-29T04:57:45.962 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 61752916~3208004 2024-03-29T04:57:46.011 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 109423546~4027903 2024-03-29T04:57:46.011 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 90099517~8018715 2024-03-29T04:57:46.065 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 56412242~706184 2024-03-29T04:57:46.072 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 76855564~7861023 2024-03-29T04:57:46.131 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 40854155~6528543 2024-03-29T04:57:46.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:45 smithi187 ceph-mon[33595]: pgmap v137: 137 pgs: 137 active+clean; 1.4 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 29 MiB/s rd, 9.4 MiB/s wr, 373 op/s 2024-03-29T04:57:46.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:45 smithi187 ceph-mon[33595]: failed to encode map e330 with expected crc 2024-03-29T04:57:46.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:45 smithi187 ceph-mon[33595]: osdmap e330: 8 total, 8 up, 8 in 2024-03-29T04:57:46.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:45 smithi187 ceph-mon[33595]: failed to encode map e330 with expected crc 2024-03-29T04:57:46.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:45 smithi187 ceph-mon[33595]: failed to encode map e330 with expected crc 2024-03-29T04:57:46.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:45 smithi187 ceph-mon[33595]: mgrmap e37: y(active, since 93s), standbys: x 2024-03-29T04:57:46.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:45 smithi187 ceph-mon[33595]: failed to encode map e330 with expected crc 2024-03-29T04:57:46.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:45 smithi187 ceph-mon[33595]: failed to encode map e330 with expected crc 2024-03-29T04:57:46.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:45 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:45.784+0000 7f1d52566700 -1 mon.b@1(peon).osd e331 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:46.162 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 100158518~2840305 2024-03-29T04:57:46.186 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 70517959~6411123 2024-03-29T04:57:46.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:45 smithi181 ceph-mon[106466]: pgmap v137: 137 pgs: 137 active+clean; 1.4 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 29 MiB/s rd, 9.4 MiB/s wr, 373 op/s 2024-03-29T04:57:46.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:45 smithi181 ceph-mon[106466]: failed to encode map e330 with expected crc 2024-03-29T04:57:46.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:45 smithi181 ceph-mon[106466]: osdmap e330: 8 total, 8 up, 8 in 2024-03-29T04:57:46.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:45 smithi181 ceph-mon[106466]: failed to encode map e330 with expected crc 2024-03-29T04:57:46.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:45 smithi181 ceph-mon[106466]: failed to encode map e330 with expected crc 2024-03-29T04:57:46.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:45 smithi181 ceph-mon[106466]: mgrmap e37: y(active, since 93s), standbys: x 2024-03-29T04:57:46.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:45 smithi181 ceph-mon[106466]: failed to encode map e330 with expected crc 2024-03-29T04:57:46.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:45 smithi181 ceph-mon[106466]: failed to encode map e330 with expected crc 2024-03-29T04:57:46.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:45 smithi181 ceph-mon[101970]: pgmap v137: 137 pgs: 137 active+clean; 1.4 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 29 MiB/s rd, 9.4 MiB/s wr, 373 op/s 2024-03-29T04:57:46.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:45 smithi181 ceph-mon[101970]: failed to encode map e330 with expected crc 2024-03-29T04:57:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:45 smithi181 ceph-mon[101970]: osdmap e330: 8 total, 8 up, 8 in 2024-03-29T04:57:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:45 smithi181 ceph-mon[101970]: failed to encode map e330 with expected crc 2024-03-29T04:57:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:45 smithi181 ceph-mon[101970]: failed to encode map e330 with expected crc 2024-03-29T04:57:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:45 smithi181 ceph-mon[101970]: mgrmap e37: y(active, since 93s), standbys: x 2024-03-29T04:57:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:45 smithi181 ceph-mon[101970]: failed to encode map e330 with expected crc 2024-03-29T04:57:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:45 smithi181 ceph-mon[101970]: failed to encode map e330 with expected crc 2024-03-29T04:57:46.226 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap2 2024-03-29T04:57:46.800 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 2 2024-03-29T04:57:47.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:46 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:46.791+0000 7f1d52566700 -1 mon.b@1(peon).osd e332 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:47.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:46 smithi187 ceph-mon[33595]: failed to encode map e331 with expected crc 2024-03-29T04:57:47.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:46 smithi187 ceph-mon[33595]: osdmap e331: 8 total, 8 up, 8 in 2024-03-29T04:57:47.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:46 smithi187 ceph-mon[33595]: failed to encode map e331 with expected crc 2024-03-29T04:57:47.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:46 smithi187 ceph-mon[33595]: failed to encode map e331 with expected crc 2024-03-29T04:57:47.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:46 smithi187 ceph-mon[33595]: failed to encode map e331 with expected crc 2024-03-29T04:57:47.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:46 smithi187 ceph-mon[33595]: failed to encode map e331 with expected crc 2024-03-29T04:57:47.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:46 smithi187 ceph-mon[33595]: from='client.? 172.21.15.181:0/674970374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:57:47.165 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:46 smithi181 ceph-mon[106466]: failed to encode map e331 with expected crc 2024-03-29T04:57:47.165 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:46 smithi181 ceph-mon[106466]: osdmap e331: 8 total, 8 up, 8 in 2024-03-29T04:57:47.165 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:46 smithi181 ceph-mon[106466]: failed to encode map e331 with expected crc 2024-03-29T04:57:47.165 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:46 smithi181 ceph-mon[106466]: failed to encode map e331 with expected crc 2024-03-29T04:57:47.165 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:46 smithi181 ceph-mon[106466]: failed to encode map e331 with expected crc 2024-03-29T04:57:47.165 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:46 smithi181 ceph-mon[106466]: failed to encode map e331 with expected crc 2024-03-29T04:57:47.166 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:46 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/674970374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:57:47.166 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:46 smithi181 ceph-mon[101970]: failed to encode map e331 with expected crc 2024-03-29T04:57:47.166 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:46 smithi181 ceph-mon[101970]: osdmap e331: 8 total, 8 up, 8 in 2024-03-29T04:57:47.166 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:46 smithi181 ceph-mon[101970]: failed to encode map e331 with expected crc 2024-03-29T04:57:47.166 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:46 smithi181 ceph-mon[101970]: failed to encode map e331 with expected crc 2024-03-29T04:57:47.166 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:46 smithi181 ceph-mon[101970]: failed to encode map e331 with expected crc 2024-03-29T04:57:47.166 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:46 smithi181 ceph-mon[101970]: failed to encode map e331 with expected crc 2024-03-29T04:57:47.166 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:46 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/674970374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:57:47.470 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap2 2024-03-29T04:57:47.480 INFO:tasks.workunit.client.0.smithi181.stdout:clone image180 -> image182 2024-03-29T04:57:47.585 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T04:57:47.634 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 173281380 2024-03-29T04:57:47.643 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 65484853~4676073 2024-03-29T04:57:47.725 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 103907028~3200107 2024-03-29T04:57:47.752 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 852510~6617477 2024-03-29T04:57:47.807 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 80118318~2613273 2024-03-29T04:57:47.808 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:47 smithi181 ceph-mon[101970]: pgmap v140: 137 pgs: 137 active+clean; 1.4 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 383 B/s wr, 0 op/s 2024-03-29T04:57:47.808 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:47 smithi181 ceph-mon[101970]: failed to encode map e332 with expected crc 2024-03-29T04:57:47.810 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:47 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:47.442+0000 7f1d52566700 -1 mon.b@1(peon).osd e333 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:47.834 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 66850790~7617666 2024-03-29T04:57:47.894 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 114685879~8156325 2024-03-29T04:57:47.899 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 161906545~2700375 2024-03-29T04:57:47.917 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 4159241~3403990 2024-03-29T04:57:47.919 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap3 2024-03-29T04:57:48.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:47 smithi187 ceph-mon[33595]: pgmap v140: 137 pgs: 137 active+clean; 1.4 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 383 B/s wr, 0 op/s 2024-03-29T04:57:48.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:47 smithi187 ceph-mon[33595]: failed to encode map e332 with expected crc 2024-03-29T04:57:48.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:47 smithi187 ceph-mon[33595]: failed to encode map e332 with expected crc 2024-03-29T04:57:48.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:47 smithi187 ceph-mon[33595]: osdmap e332: 8 total, 8 up, 8 in 2024-03-29T04:57:48.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:47 smithi187 ceph-mon[33595]: failed to encode map e332 with expected crc 2024-03-29T04:57:48.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:47 smithi187 ceph-mon[33595]: failed to encode map e332 with expected crc 2024-03-29T04:57:48.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:47 smithi187 ceph-mon[33595]: osdmap e333: 8 total, 8 up, 8 in 2024-03-29T04:57:48.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:47 smithi187 ceph-mon[33595]: failed to encode map e333 with expected crc 2024-03-29T04:57:48.134 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:47 smithi181 ceph-mon[101970]: failed to encode map e332 with expected crc 2024-03-29T04:57:48.134 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:47 smithi181 ceph-mon[101970]: osdmap e332: 8 total, 8 up, 8 in 2024-03-29T04:57:48.134 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:47 smithi181 ceph-mon[101970]: failed to encode map e332 with expected crc 2024-03-29T04:57:48.134 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:47 smithi181 ceph-mon[101970]: failed to encode map e332 with expected crc 2024-03-29T04:57:48.135 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:47 smithi181 ceph-mon[101970]: osdmap e333: 8 total, 8 up, 8 in 2024-03-29T04:57:48.135 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:47 smithi181 ceph-mon[101970]: failed to encode map e333 with expected crc 2024-03-29T04:57:48.135 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:47 smithi181 ceph-mon[106466]: pgmap v140: 137 pgs: 137 active+clean; 1.4 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 383 B/s wr, 0 op/s 2024-03-29T04:57:48.135 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:47 smithi181 ceph-mon[106466]: failed to encode map e332 with expected crc 2024-03-29T04:57:48.135 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:47 smithi181 ceph-mon[106466]: failed to encode map e332 with expected crc 2024-03-29T04:57:48.135 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:47 smithi181 ceph-mon[106466]: osdmap e332: 8 total, 8 up, 8 in 2024-03-29T04:57:48.135 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:47 smithi181 ceph-mon[106466]: failed to encode map e332 with expected crc 2024-03-29T04:57:48.135 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:47 smithi181 ceph-mon[106466]: failed to encode map e332 with expected crc 2024-03-29T04:57:48.135 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:47 smithi181 ceph-mon[106466]: osdmap e333: 8 total, 8 up, 8 in 2024-03-29T04:57:48.136 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:47 smithi181 ceph-mon[106466]: failed to encode map e333 with expected crc 2024-03-29T04:57:48.645 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:48.809 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:57:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:57:48.537+0000 7f67753a3700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-03-29T04:57:49.100 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:57:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:57:48] "GET /metrics HTTP/1.1" 200 38723 "" "Prometheus/2.43.0" 2024-03-29T04:57:49.100 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:48 smithi181 ceph-mon[101970]: failed to encode map e333 with expected crc 2024-03-29T04:57:49.100 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:48 smithi181 ceph-mon[101970]: failed to encode map e333 with expected crc 2024-03-29T04:57:49.101 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:48 smithi181 ceph-mon[101970]: failed to encode map e333 with expected crc 2024-03-29T04:57:49.101 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:48 smithi181 ceph-mon[106466]: failed to encode map e333 with expected crc 2024-03-29T04:57:49.101 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:48 smithi181 ceph-mon[106466]: failed to encode map e333 with expected crc 2024-03-29T04:57:49.101 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:48 smithi181 ceph-mon[106466]: failed to encode map e333 with expected crc 2024-03-29T04:57:49.133 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:57:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:57:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:57:49.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:48 smithi187 ceph-mon[33595]: failed to encode map e333 with expected crc 2024-03-29T04:57:49.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:48 smithi187 ceph-mon[33595]: failed to encode map e333 with expected crc 2024-03-29T04:57:49.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:48 smithi187 ceph-mon[33595]: failed to encode map e333 with expected crc 2024-03-29T04:57:49.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:48.806+0000 7f1d52566700 -1 mon.b@1(peon).osd e334 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:49.844 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap3 2024-03-29T04:57:49.854 INFO:tasks.workunit.client.0.smithi181.stdout:clone image182 -> image184 2024-03-29T04:57:50.036 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:57:50.093 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:49 smithi187 ceph-mon[33595]: pgmap v143: 137 pgs: 137 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.4 MiB/s rd, 15 MiB/s wr, 242 op/s 2024-03-29T04:57:50.094 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:49 smithi187 ceph-mon[33595]: failed to encode map e334 with expected crc 2024-03-29T04:57:50.094 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:49 smithi187 ceph-mon[33595]: osdmap e334: 8 total, 8 up, 8 in 2024-03-29T04:57:50.094 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:49 smithi187 ceph-mon[33595]: failed to encode map e334 with expected crc 2024-03-29T04:57:50.094 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:49 smithi187 ceph-mon[33595]: failed to encode map e334 with expected crc 2024-03-29T04:57:50.094 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:49 smithi187 ceph-mon[33595]: failed to encode map e334 with expected crc 2024-03-29T04:57:50.094 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:49 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:50.094 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:49 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:50.094 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:49 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:49.817+0000 7f1d52566700 -1 mon.b@1(peon).osd e335 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:50.160 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:49 smithi181 ceph-mon[106466]: pgmap v143: 137 pgs: 137 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.4 MiB/s rd, 15 MiB/s wr, 242 op/s 2024-03-29T04:57:50.160 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:49 smithi181 ceph-mon[106466]: failed to encode map e334 with expected crc 2024-03-29T04:57:50.161 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:49 smithi181 ceph-mon[106466]: osdmap e334: 8 total, 8 up, 8 in 2024-03-29T04:57:50.161 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:49 smithi181 ceph-mon[106466]: failed to encode map e334 with expected crc 2024-03-29T04:57:50.161 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:49 smithi181 ceph-mon[106466]: failed to encode map e334 with expected crc 2024-03-29T04:57:50.161 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:49 smithi181 ceph-mon[106466]: failed to encode map e334 with expected crc 2024-03-29T04:57:50.161 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:49 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:50.161 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:49 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:50.161 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:49 smithi181 ceph-mon[101970]: pgmap v143: 137 pgs: 137 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.4 MiB/s rd, 15 MiB/s wr, 242 op/s 2024-03-29T04:57:50.161 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:49 smithi181 ceph-mon[101970]: failed to encode map e334 with expected crc 2024-03-29T04:57:50.162 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:49 smithi181 ceph-mon[101970]: osdmap e334: 8 total, 8 up, 8 in 2024-03-29T04:57:50.162 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:49 smithi181 ceph-mon[101970]: failed to encode map e334 with expected crc 2024-03-29T04:57:50.162 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:49 smithi181 ceph-mon[101970]: failed to encode map e334 with expected crc 2024-03-29T04:57:50.162 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:49 smithi181 ceph-mon[101970]: failed to encode map e334 with expected crc 2024-03-29T04:57:50.162 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:49 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:50.162 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:49 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:50.383 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:57:50 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:57:50.087Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", public_addr=\"172.21.15.181\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", public_addr=\"172.21.15.181\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-03-29T04:57:50.601 INFO:tasks.workunit.client.0.smithi181.stdout:write: a 77458979~4366313 2024-03-29T04:57:50.673 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 88303906~848263 2024-03-29T04:57:50.706 INFO:tasks.workunit.client.0.smithi181.stdout:write: a 91476154~2232162 2024-03-29T04:57:50.745 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 27628348~1499355 2024-03-29T04:57:50.762 INFO:tasks.workunit.client.0.smithi181.stdout:write: a 86663810~7493753 2024-03-29T04:57:50.815 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 166739711~6541669 2024-03-29T04:57:50.840 INFO:tasks.workunit.client.0.smithi181.stdout:write: a 92973102~3205286 2024-03-29T04:57:50.859 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 154770505~3663532 2024-03-29T04:57:50.888 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: user0 2024-03-29T04:57:50.914 INFO:teuthology.orchestra.run.smithi181.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T04:57:50.914 INFO:teuthology.orchestra.run.smithi181.stdout:alertmanager.a smithi181 *:9093,9094 running (3m) 1s ago 7m 27.6M - 0.25.0 c8568f914cd2 69e0ad67b57f 2024-03-29T04:57:50.914 INFO:teuthology.orchestra.run.smithi181.stdout:grafana.a smithi187 *:3000 error 24s ago 6m - - 2024-03-29T04:57:50.914 INFO:teuthology.orchestra.run.smithi181.stdout:mds.foo.smithi181.pflrwm smithi181 running (5m) 1s ago 5m 16.4M - 17.2.7-789-g4213e17c cf22eb8683be 0f200e33dce6 2024-03-29T04:57:50.914 INFO:teuthology.orchestra.run.smithi181.stdout:mds.foo.smithi187.vcsnqx smithi187 running (5m) 24s ago 5m 16.5M - 17.2.7-789-g4213e17c cf22eb8683be 245621ed14ed 2024-03-29T04:57:50.914 INFO:teuthology.orchestra.run.smithi181.stdout:mgr.x smithi187 *:8443,9283,8765 running (2m) 24s ago 11m 449M - 19.0.0-1959-g8da23624 54112f57da44 19bc315720bd 2024-03-29T04:57:50.914 INFO:teuthology.orchestra.run.smithi181.stdout:mgr.y smithi181 *:8443,9283,8765 running (2m) 1s ago 12m 501M - 19.0.0-1959-g8da23624 54112f57da44 b07d84f4244a 2024-03-29T04:57:50.914 INFO:teuthology.orchestra.run.smithi181.stdout:mon.a smithi181 running (108s) 1s ago 13m 44.7M 2048M 19.0.0-1959-g8da23624 54112f57da44 4e5d3325d991 2024-03-29T04:57:50.914 INFO:teuthology.orchestra.run.smithi181.stdout:mon.b smithi187 running (11m) 24s ago 11m 74.2M 2048M 17.2.7-789-g4213e17c cf22eb8683be a376bfeefb23 2024-03-29T04:57:50.914 INFO:teuthology.orchestra.run.smithi181.stdout:mon.c smithi181 running (12s) 1s ago 11m 28.3M 2048M 19.0.0-1959-g8da23624 54112f57da44 10f02d533230 2024-03-29T04:57:50.914 INFO:teuthology.orchestra.run.smithi181.stdout:node-exporter.a smithi181 *:9100 running (3m) 1s ago 6m 15.0M - 1.5.0 0da6a335fe13 4029d029600b 2024-03-29T04:57:50.915 INFO:teuthology.orchestra.run.smithi181.stdout:node-exporter.b smithi187 *:9100 running (2m) 24s ago 6m 16.0M - 1.5.0 0da6a335fe13 3894f6e97d55 2024-03-29T04:57:50.915 INFO:teuthology.orchestra.run.smithi181.stdout:osd.0 smithi181 running (10m) 1s ago 10m 148M 2480M 17.2.7-789-g4213e17c cf22eb8683be ae426d29a1a8 2024-03-29T04:57:50.915 INFO:teuthology.orchestra.run.smithi181.stdout:osd.1 smithi181 running (10m) 1s ago 10m 118M 2480M 17.2.7-789-g4213e17c cf22eb8683be 7ed1966c65cf 2024-03-29T04:57:50.915 INFO:teuthology.orchestra.run.smithi181.stdout:osd.2 smithi181 running (9m) 1s ago 9m 114M 2480M 17.2.7-789-g4213e17c cf22eb8683be 9c61fb0d684c 2024-03-29T04:57:50.915 INFO:teuthology.orchestra.run.smithi181.stdout:osd.3 smithi181 running (9m) 1s ago 9m 115M 2480M 17.2.7-789-g4213e17c cf22eb8683be 6b0bf471cfcb 2024-03-29T04:57:50.915 INFO:teuthology.orchestra.run.smithi181.stdout:osd.4 smithi187 running (9m) 24s ago 9m 135M 2480M 17.2.7-789-g4213e17c cf22eb8683be 42698769ef84 2024-03-29T04:57:50.915 INFO:teuthology.orchestra.run.smithi181.stdout:osd.5 smithi187 running (8m) 24s ago 8m 107M 2480M 17.2.7-789-g4213e17c cf22eb8683be 9599ff85a9e7 2024-03-29T04:57:50.915 INFO:teuthology.orchestra.run.smithi181.stdout:osd.6 smithi187 running (8m) 24s ago 8m 158M 2480M 17.2.7-789-g4213e17c cf22eb8683be c3287470c1fa 2024-03-29T04:57:50.915 INFO:teuthology.orchestra.run.smithi181.stdout:osd.7 smithi187 running (7m) 24s ago 7m 110M 2480M 17.2.7-789-g4213e17c cf22eb8683be 89fb8ccf8705 2024-03-29T04:57:50.915 INFO:teuthology.orchestra.run.smithi181.stdout:prometheus.a smithi187 *:9095 running (2m) 24s ago 7m 52.7M - 2.43.0 a07b618ecd1d 882b8922d74b 2024-03-29T04:57:51.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:50 smithi187 ceph-mon[33595]: failed to encode map e335 with expected crc 2024-03-29T04:57:51.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:50 smithi187 ceph-mon[33595]: failed to encode map e335 with expected crc 2024-03-29T04:57:51.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:50 smithi187 ceph-mon[33595]: osdmap e335: 8 total, 8 up, 8 in 2024-03-29T04:57:51.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:50 smithi187 ceph-mon[33595]: failed to encode map e335 with expected crc 2024-03-29T04:57:51.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:50 smithi187 ceph-mon[33595]: failed to encode map e335 with expected crc 2024-03-29T04:57:51.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:50 smithi187 ceph-mon[33595]: failed to encode map e335 with expected crc 2024-03-29T04:57:51.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:50 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:51.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:50 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:51.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:50 smithi181 ceph-mon[106466]: failed to encode map e335 with expected crc 2024-03-29T04:57:51.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:50 smithi181 ceph-mon[106466]: failed to encode map e335 with expected crc 2024-03-29T04:57:51.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:50 smithi181 ceph-mon[106466]: osdmap e335: 8 total, 8 up, 8 in 2024-03-29T04:57:51.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:50 smithi181 ceph-mon[106466]: failed to encode map e335 with expected crc 2024-03-29T04:57:51.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:50 smithi181 ceph-mon[106466]: failed to encode map e335 with expected crc 2024-03-29T04:57:51.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:50 smithi181 ceph-mon[106466]: failed to encode map e335 with expected crc 2024-03-29T04:57:51.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:50 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:51.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:50 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:51.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:50 smithi181 ceph-mon[101970]: failed to encode map e335 with expected crc 2024-03-29T04:57:51.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:50 smithi181 ceph-mon[101970]: failed to encode map e335 with expected crc 2024-03-29T04:57:51.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:50 smithi181 ceph-mon[101970]: osdmap e335: 8 total, 8 up, 8 in 2024-03-29T04:57:51.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:50 smithi181 ceph-mon[101970]: failed to encode map e335 with expected crc 2024-03-29T04:57:51.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:50 smithi181 ceph-mon[101970]: failed to encode map e335 with expected crc 2024-03-29T04:57:51.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:50 smithi181 ceph-mon[101970]: failed to encode map e335 with expected crc 2024-03-29T04:57:51.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:50 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:51.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:50 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:51.702 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade pause' 2024-03-29T04:57:51.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:51 smithi181 ceph-mon[106466]: pgmap v146: 137 pgs: 137 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.4 MiB/s rd, 15 MiB/s wr, 242 op/s 2024-03-29T04:57:51.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:51 smithi181 ceph-mon[106466]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:57:51.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:51 smithi181 ceph-mon[101970]: pgmap v146: 137 pgs: 137 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.4 MiB/s rd, 15 MiB/s wr, 242 op/s 2024-03-29T04:57:51.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:51 smithi181 ceph-mon[101970]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:57:52.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:51 smithi187 ceph-mon[33595]: pgmap v146: 137 pgs: 137 active+clean; 1.5 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 3.4 MiB/s rd, 15 MiB/s wr, 242 op/s 2024-03-29T04:57:52.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:51 smithi187 ceph-mon[33595]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:57:52.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:51 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:51.829+0000 7f1d52566700 -1 mon.b@1(peon).osd e336 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:52.854 INFO:tasks.workunit.client.0.smithi181.stdout:write: b 87515928~1892402 2024-03-29T04:57:52.862 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 40566792~2677807 2024-03-29T04:57:52.862 INFO:tasks.workunit.client.0.smithi181.stdout:write: b 120109802~2858906 2024-03-29T04:57:52.867 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 136702277~3152021 2024-03-29T04:57:52.867 INFO:tasks.workunit.client.0.smithi181.stdout:write: b 154040126~4870553 2024-03-29T04:57:52.873 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 391630~5638984 2024-03-29T04:57:52.875 INFO:tasks.workunit.client.0.smithi181.stdout:write: b 144361504~4699171 2024-03-29T04:57:52.917 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:52 smithi181 ceph-mon[106466]: failed to encode map e336 with expected crc 2024-03-29T04:57:52.917 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:52 smithi181 ceph-mon[106466]: osdmap e336: 8 total, 8 up, 8 in 2024-03-29T04:57:52.917 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:52 smithi181 ceph-mon[106466]: failed to encode map e336 with expected crc 2024-03-29T04:57:52.917 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:52 smithi181 ceph-mon[106466]: failed to encode map e336 with expected crc 2024-03-29T04:57:52.917 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:52 smithi181 ceph-mon[106466]: failed to encode map e336 with expected crc 2024-03-29T04:57:52.917 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:52 smithi181 ceph-mon[106466]: failed to encode map e336 with expected crc 2024-03-29T04:57:52.917 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:52 smithi181 ceph-mon[101970]: failed to encode map e336 with expected crc 2024-03-29T04:57:52.917 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:52 smithi181 ceph-mon[101970]: osdmap e336: 8 total, 8 up, 8 in 2024-03-29T04:57:52.917 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:52 smithi181 ceph-mon[101970]: failed to encode map e336 with expected crc 2024-03-29T04:57:52.917 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:52 smithi181 ceph-mon[101970]: failed to encode map e336 with expected crc 2024-03-29T04:57:52.918 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:52 smithi181 ceph-mon[101970]: failed to encode map e336 with expected crc 2024-03-29T04:57:52.918 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:52 smithi181 ceph-mon[101970]: failed to encode map e336 with expected crc 2024-03-29T04:57:52.988 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 103945474~4508388 2024-03-29T04:57:52.999 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: user1 2024-03-29T04:57:53.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:52 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:52.831+0000 7f1d52566700 -1 mon.b@1(peon).osd e337 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:53.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:52 smithi187 ceph-mon[33595]: failed to encode map e336 with expected crc 2024-03-29T04:57:53.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:52 smithi187 ceph-mon[33595]: osdmap e336: 8 total, 8 up, 8 in 2024-03-29T04:57:53.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:52 smithi187 ceph-mon[33595]: failed to encode map e336 with expected crc 2024-03-29T04:57:53.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:52 smithi187 ceph-mon[33595]: failed to encode map e336 with expected crc 2024-03-29T04:57:53.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:52 smithi187 ceph-mon[33595]: failed to encode map e336 with expected crc 2024-03-29T04:57:53.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:52 smithi187 ceph-mon[33595]: failed to encode map e336 with expected crc 2024-03-29T04:57:53.199 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:53.909 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:53 smithi181 ceph-mon[101970]: pgmap v148: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 30 MiB/s rd, 27 MiB/s wr, 707 op/s 2024-03-29T04:57:53.909 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:53 smithi181 ceph-mon[101970]: failed to encode map e337 with expected crc 2024-03-29T04:57:53.910 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:53 smithi181 ceph-mon[101970]: osdmap e337: 8 total, 8 up, 8 in 2024-03-29T04:57:53.910 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:53 smithi181 ceph-mon[101970]: failed to encode map e337 with expected crc 2024-03-29T04:57:53.910 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:53 smithi181 ceph-mon[101970]: failed to encode map e337 with expected crc 2024-03-29T04:57:53.910 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:53 smithi181 ceph-mon[106466]: pgmap v148: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 30 MiB/s rd, 27 MiB/s wr, 707 op/s 2024-03-29T04:57:53.910 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:53 smithi181 ceph-mon[106466]: failed to encode map e337 with expected crc 2024-03-29T04:57:53.910 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:53 smithi181 ceph-mon[106466]: osdmap e337: 8 total, 8 up, 8 in 2024-03-29T04:57:53.910 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:53 smithi181 ceph-mon[106466]: failed to encode map e337 with expected crc 2024-03-29T04:57:53.910 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:53 smithi181 ceph-mon[106466]: failed to encode map e337 with expected crc 2024-03-29T04:57:54.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:53 smithi187 ceph-mon[33595]: pgmap v148: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 30 MiB/s rd, 27 MiB/s wr, 707 op/s 2024-03-29T04:57:54.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:53 smithi187 ceph-mon[33595]: failed to encode map e337 with expected crc 2024-03-29T04:57:54.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:53 smithi187 ceph-mon[33595]: osdmap e337: 8 total, 8 up, 8 in 2024-03-29T04:57:54.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:53 smithi187 ceph-mon[33595]: failed to encode map e337 with expected crc 2024-03-29T04:57:54.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:53 smithi187 ceph-mon[33595]: failed to encode map e337 with expected crc 2024-03-29T04:57:54.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:53 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:53.840+0000 7f1d52566700 -1 mon.b@1(peon).osd e338 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:54.866 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 254825678 2024-03-29T04:57:54.877 INFO:tasks.workunit.client.0.smithi181.stdout:write: c 57049961~2634952 2024-03-29T04:57:54.885 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 8091234~4666793 2024-03-29T04:57:54.886 INFO:tasks.workunit.client.0.smithi181.stdout:write: c 218176737~8050870 2024-03-29T04:57:54.896 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 18923509~3250538 2024-03-29T04:57:54.897 INFO:tasks.workunit.client.0.smithi181.stdout:write: c 203125120~4391922 2024-03-29T04:57:54.903 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 55734113~5983506 2024-03-29T04:57:54.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[101970]: failed to encode map e338 with expected crc 2024-03-29T04:57:54.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[101970]: osdmap e338: 8 total, 8 up, 8 in 2024-03-29T04:57:54.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[101970]: failed to encode map e338 with expected crc 2024-03-29T04:57:54.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[101970]: failed to encode map e338 with expected crc 2024-03-29T04:57:54.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[101970]: failed to encode map e338 with expected crc 2024-03-29T04:57:54.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:54.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:54.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:57:54.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:57:54.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:54.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:57:54.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:57:54.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:57:54.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-03-29T04:57:54.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[106466]: failed to encode map e338 with expected crc 2024-03-29T04:57:54.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[106466]: osdmap e338: 8 total, 8 up, 8 in 2024-03-29T04:57:54.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[106466]: failed to encode map e338 with expected crc 2024-03-29T04:57:54.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[106466]: failed to encode map e338 with expected crc 2024-03-29T04:57:54.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[106466]: failed to encode map e338 with expected crc 2024-03-29T04:57:54.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:54.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:54.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:57:54.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:57:54.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:54.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:57:54.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:57:54.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:57:54.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:54 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-03-29T04:57:55.040 INFO:tasks.workunit.client.0.smithi181.stdout:write: c 219644840~6144207 2024-03-29T04:57:55.046 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 58979982~6038555 2024-03-29T04:57:55.046 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: user2 2024-03-29T04:57:55.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:54 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:54.843+0000 7f1d52566700 -1 mon.b@1(peon).osd e339 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:55.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:54 smithi187 ceph-mon[33595]: failed to encode map e338 with expected crc 2024-03-29T04:57:55.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:54 smithi187 ceph-mon[33595]: osdmap e338: 8 total, 8 up, 8 in 2024-03-29T04:57:55.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:54 smithi187 ceph-mon[33595]: failed to encode map e338 with expected crc 2024-03-29T04:57:55.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:54 smithi187 ceph-mon[33595]: failed to encode map e338 with expected crc 2024-03-29T04:57:55.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:54 smithi187 ceph-mon[33595]: failed to encode map e338 with expected crc 2024-03-29T04:57:55.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:54 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:55.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:54 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:55.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:54 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:57:55.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:54 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:57:55.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:54 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:55.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:54 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:57:55.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:54 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:57:55.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:54 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:57:55.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:54 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-03-29T04:57:55.892 INFO:teuthology.orchestra.run.smithi181.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:57:56.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:55 smithi187 ceph-mon[33595]: pgmap v151: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 32 MiB/s rd, 29 MiB/s wr, 763 op/s 2024-03-29T04:57:56.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:55 smithi187 ceph-mon[33595]: pgmap v152: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 37 MiB/s rd, 34 MiB/s wr, 873 op/s 2024-03-29T04:57:56.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:55 smithi187 ceph-mon[33595]: pgmap v153: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 53 MiB/s rd, 49 MiB/s wr, 1.26k op/s 2024-03-29T04:57:56.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:55 smithi187 ceph-mon[33595]: Upgrade: It appears safe to stop mon.b 2024-03-29T04:57:56.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:55 smithi187 ceph-mon[33595]: failed to encode map e339 with expected crc 2024-03-29T04:57:56.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:55 smithi187 ceph-mon[33595]: osdmap e339: 8 total, 8 up, 8 in 2024-03-29T04:57:56.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:55 smithi187 ceph-mon[33595]: failed to encode map e339 with expected crc 2024-03-29T04:57:56.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:55 smithi187 ceph-mon[33595]: failed to encode map e339 with expected crc 2024-03-29T04:57:56.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:55.849+0000 7f1d52566700 -1 mon.b@1(peon).osd e340 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:56.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:55 smithi181 ceph-mon[106466]: pgmap v151: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 32 MiB/s rd, 29 MiB/s wr, 763 op/s 2024-03-29T04:57:56.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:55 smithi181 ceph-mon[106466]: pgmap v152: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 37 MiB/s rd, 34 MiB/s wr, 873 op/s 2024-03-29T04:57:56.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:55 smithi181 ceph-mon[106466]: pgmap v153: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 53 MiB/s rd, 49 MiB/s wr, 1.26k op/s 2024-03-29T04:57:56.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:55 smithi181 ceph-mon[106466]: Upgrade: It appears safe to stop mon.b 2024-03-29T04:57:56.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:55 smithi181 ceph-mon[106466]: failed to encode map e339 with expected crc 2024-03-29T04:57:56.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:55 smithi181 ceph-mon[106466]: osdmap e339: 8 total, 8 up, 8 in 2024-03-29T04:57:56.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:55 smithi181 ceph-mon[106466]: failed to encode map e339 with expected crc 2024-03-29T04:57:56.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:55 smithi181 ceph-mon[106466]: failed to encode map e339 with expected crc 2024-03-29T04:57:56.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:55 smithi181 ceph-mon[101970]: pgmap v151: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 32 MiB/s rd, 29 MiB/s wr, 763 op/s 2024-03-29T04:57:56.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:55 smithi181 ceph-mon[101970]: pgmap v152: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 37 MiB/s rd, 34 MiB/s wr, 873 op/s 2024-03-29T04:57:56.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:55 smithi181 ceph-mon[101970]: pgmap v153: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 53 MiB/s rd, 49 MiB/s wr, 1.26k op/s 2024-03-29T04:57:56.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:55 smithi181 ceph-mon[101970]: Upgrade: It appears safe to stop mon.b 2024-03-29T04:57:56.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:55 smithi181 ceph-mon[101970]: failed to encode map e339 with expected crc 2024-03-29T04:57:56.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:55 smithi181 ceph-mon[101970]: osdmap e339: 8 total, 8 up, 8 in 2024-03-29T04:57:56.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:55 smithi181 ceph-mon[101970]: failed to encode map e339 with expected crc 2024-03-29T04:57:56.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:55 smithi181 ceph-mon[101970]: failed to encode map e339 with expected crc 2024-03-29T04:57:56.636 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'sleep 60' 2024-03-29T04:57:56.879 INFO:tasks.workunit.client.0.smithi181.stdout:write: d 90597128~508854 2024-03-29T04:57:56.884 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 210231635~6023625 2024-03-29T04:57:56.884 INFO:tasks.workunit.client.0.smithi181.stdout:write: d 12101059~7474137 2024-03-29T04:57:56.888 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 87221148~8382831 2024-03-29T04:57:56.914 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:56.923 INFO:tasks.workunit.client.0.smithi181.stdout:write: d 53059559~2250427 2024-03-29T04:57:56.927 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 122147461~3735992 2024-03-29T04:57:56.929 INFO:tasks.workunit.client.0.smithi181.stdout:write: d 162005838~6483542 2024-03-29T04:57:56.935 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 99315347~5288170 2024-03-29T04:57:57.023 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: user3 2024-03-29T04:57:57.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:56.849+0000 7f1d52566700 -1 mon.b@1(peon).osd e341 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:56 smithi187 ceph-mon[33595]: failed to encode map e340 with expected crc 2024-03-29T04:57:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:56 smithi187 ceph-mon[33595]: osdmap e340: 8 total, 8 up, 8 in 2024-03-29T04:57:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:56 smithi187 ceph-mon[33595]: failed to encode map e340 with expected crc 2024-03-29T04:57:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:56 smithi187 ceph-mon[33595]: failed to encode map e340 with expected crc 2024-03-29T04:57:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:56 smithi187 ceph-mon[33595]: failed to encode map e340 with expected crc 2024-03-29T04:57:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:56 smithi187 ceph-mon[33595]: failed to encode map e340 with expected crc 2024-03-29T04:57:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:56 smithi187 ceph-mon[33595]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:57:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:56 smithi187 ceph-mon[33595]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:57:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:56 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:56 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:56 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:56 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:57:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:56 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:57:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:56 smithi187 ceph-mon[33595]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:57:57.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[101970]: failed to encode map e340 with expected crc 2024-03-29T04:57:57.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[101970]: osdmap e340: 8 total, 8 up, 8 in 2024-03-29T04:57:57.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[101970]: failed to encode map e340 with expected crc 2024-03-29T04:57:57.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[101970]: failed to encode map e340 with expected crc 2024-03-29T04:57:57.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[101970]: failed to encode map e340 with expected crc 2024-03-29T04:57:57.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[101970]: failed to encode map e340 with expected crc 2024-03-29T04:57:57.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[101970]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:57:57.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[101970]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:57:57.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:57.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:57.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:57.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:57:57.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:57:57.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:57:57.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[106466]: failed to encode map e340 with expected crc 2024-03-29T04:57:57.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[106466]: osdmap e340: 8 total, 8 up, 8 in 2024-03-29T04:57:57.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[106466]: failed to encode map e340 with expected crc 2024-03-29T04:57:57.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[106466]: failed to encode map e340 with expected crc 2024-03-29T04:57:57.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[106466]: failed to encode map e340 with expected crc 2024-03-29T04:57:57.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[106466]: failed to encode map e340 with expected crc 2024-03-29T04:57:57.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[106466]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:57:57.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[106466]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:57:57.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:57.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:57.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:57:57.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:57:57.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:57:57.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:57:58.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:57 smithi187 ceph-mon[33595]: Upgrade: Updating mon.b 2024-03-29T04:57:58.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:57 smithi187 ceph-mon[33595]: Deploying daemon mon.b on smithi187 2024-03-29T04:57:58.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:57 smithi187 ceph-mon[33595]: pgmap v156: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-03-29T04:57:58.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:57 smithi187 ceph-mon[33595]: failed to encode map e341 with expected crc 2024-03-29T04:57:58.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:57 smithi187 ceph-mon[33595]: osdmap e341: 8 total, 8 up, 8 in 2024-03-29T04:57:58.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:57 smithi187 ceph-mon[33595]: failed to encode map e341 with expected crc 2024-03-29T04:57:58.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:57 smithi187 ceph-mon[33595]: failed to encode map e341 with expected crc 2024-03-29T04:57:58.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:57 smithi187 ceph-mon[33595]: failed to encode map e341 with expected crc 2024-03-29T04:57:58.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:57 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:57.863+0000 7f1d52566700 -1 mon.b@1(peon).osd e342 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:57:58.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:57 smithi181 ceph-mon[101970]: Upgrade: Updating mon.b 2024-03-29T04:57:58.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:57 smithi181 ceph-mon[101970]: Deploying daemon mon.b on smithi187 2024-03-29T04:57:58.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:57 smithi181 ceph-mon[101970]: pgmap v156: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-03-29T04:57:58.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:57 smithi181 ceph-mon[101970]: failed to encode map e341 with expected crc 2024-03-29T04:57:58.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:57 smithi181 ceph-mon[101970]: osdmap e341: 8 total, 8 up, 8 in 2024-03-29T04:57:58.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:57 smithi181 ceph-mon[101970]: failed to encode map e341 with expected crc 2024-03-29T04:57:58.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:57 smithi181 ceph-mon[101970]: failed to encode map e341 with expected crc 2024-03-29T04:57:58.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:57:57 smithi181 ceph-mon[101970]: failed to encode map e341 with expected crc 2024-03-29T04:57:58.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:57 smithi181 ceph-mon[106466]: Upgrade: Updating mon.b 2024-03-29T04:57:58.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:57 smithi181 ceph-mon[106466]: Deploying daemon mon.b on smithi187 2024-03-29T04:57:58.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:57 smithi181 ceph-mon[106466]: pgmap v156: 137 pgs: 137 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-03-29T04:57:58.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:57 smithi181 ceph-mon[106466]: failed to encode map e341 with expected crc 2024-03-29T04:57:58.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:57 smithi181 ceph-mon[106466]: osdmap e341: 8 total, 8 up, 8 in 2024-03-29T04:57:58.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:57 smithi181 ceph-mon[106466]: failed to encode map e341 with expected crc 2024-03-29T04:57:58.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:57 smithi181 ceph-mon[106466]: failed to encode map e341 with expected crc 2024-03-29T04:57:58.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:57:57 smithi181 ceph-mon[106466]: failed to encode map e341 with expected crc 2024-03-29T04:57:58.832 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:58 smithi187 systemd[1]: Stopping Ceph mon.b for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:57:58.833 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:57:58 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:57:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:57:59.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:58 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:58.825+0000 7f1d57d71700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:57:59.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:58 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[33563]: 2024-03-29T04:57:58.825+0000 7f1d57d71700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-03-29T04:57:59.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:58 smithi187 podman[93335]: 2024-03-29 04:57:58.998995464 +0000 UTC m=+0.290357423 container died a376bfeefb23a8f9d7b1af4b814f9bc75efa5c4e47a708290a8008b8d1b970ed (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:57:59.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:57:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:57:58] "GET /metrics HTTP/1.1" 200 38721 "" "Prometheus/2.43.0" 2024-03-29T04:57:59.502 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:59 smithi187 podman[93335]: 2024-03-29 04:57:59.380749439 +0000 UTC m=+0.672111381 container cleanup a376bfeefb23a8f9d7b1af4b814f9bc75efa5c4e47a708290a8008b8d1b970ed (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True) 2024-03-29T04:57:59.502 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:59 smithi187 bash[93335]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b 2024-03-29T04:57:59.824 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:59 smithi187 podman[93354]: 2024-03-29 04:57:59.649103519 +0000 UTC m=+0.645270438 container remove a376bfeefb23a8f9d7b1af4b814f9bc75efa5c4e47a708290a8008b8d1b970ed (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, io.buildah.version=1.33.5) 2024-03-29T04:58:00.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:59 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.b.service: Deactivated successfully. 2024-03-29T04:58:00.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:59 smithi187 systemd[1]: Stopped Ceph mon.b for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:58:00.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:57:59 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.b.service: Consumed 10.491s CPU time. 2024-03-29T04:58:00.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 systemd[1]: Starting Ceph mon.b for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:58:00.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 podman[93498]: 2024-03-29 04:58:00.1599696 +0000 UTC m=+0.018116403 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:58:00.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 podman[93498]: 2024-03-29 04:58:00.284043477 +0000 UTC m=+0.142190277 container create 4de0ae3c44c36f2dc00ac3f31d42bb83b5867282720d6b8baf7aa7fb7949eea5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:58:01.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 podman[93498]: 2024-03-29 04:58:00.637267352 +0000 UTC m=+0.495414160 container init 4de0ae3c44c36f2dc00ac3f31d42bb83b5867282720d6b8baf7aa7fb7949eea5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-8da2362) 2024-03-29T04:58:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 podman[93498]: 2024-03-29 04:58:00.642829133 +0000 UTC m=+0.500975934 container start 4de0ae3c44c36f2dc00ac3f31d42bb83b5867282720d6b8baf7aa7fb7949eea5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-03-29T04:58:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: set uid:gid to 167:167 (ceph:ceph) 2024-03-29T04:58:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev), process ceph-mon, pid 7 2024-03-29T04:58:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: pidfile_write: ignore empty --pid-file 2024-03-29T04:58:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: load: jerasure load: lrc 2024-03-29T04:58:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: RocksDB version: 7.9.2 2024-03-29T04:58:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Git sha 0 2024-03-29T04:58:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Compile date 2024-03-28 15:35:57 2024-03-29T04:58:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: DB SUMMARY 2024-03-29T04:58:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: DB Session ID: GP2DFNGQVQUS41VD359A 2024-03-29T04:58:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: CURRENT file: CURRENT 2024-03-29T04:58:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: IDENTITY file: IDENTITY 2024-03-29T04:58:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1099 Bytes 2024-03-29T04:58:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-03-29T04:58:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 11220527 ; 2024-03-29T04:58:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.error_if_exists: 0 2024-03-29T04:58:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.create_if_missing: 0 2024-03-29T04:58:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.paranoid_checks: 1 2024-03-29T04:58:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-29T04:58:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-29T04:58:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-29T04:58:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.env: 0x555b0358ac60 2024-03-29T04:58:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.fs: PosixFileSystem 2024-03-29T04:58:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.info_log: 0x555b050a3ae0 2024-03-29T04:58:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_file_opening_threads: 16 2024-03-29T04:58:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.statistics: (nil) 2024-03-29T04:58:01.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.use_fsync: 0 2024-03-29T04:58:01.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_log_file_size: 0 2024-03-29T04:58:01.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-29T04:58:01.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-29T04:58:01.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.keep_log_file_num: 1000 2024-03-29T04:58:01.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.recycle_log_file_num: 0 2024-03-29T04:58:01.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.allow_fallocate: 1 2024-03-29T04:58:01.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.allow_mmap_reads: 0 2024-03-29T04:58:01.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.allow_mmap_writes: 0 2024-03-29T04:58:01.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.use_direct_reads: 0 2024-03-29T04:58:01.138 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-29T04:58:01.138 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.create_missing_column_families: 0 2024-03-29T04:58:01.138 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.db_log_dir: 2024-03-29T04:58:01.138 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.wal_dir: 2024-03-29T04:58:01.138 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-29T04:58:01.138 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-29T04:58:01.138 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-29T04:58:01.138 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-29T04:58:01.138 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-29T04:58:01.138 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-29T04:58:01.138 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.advise_random_on_open: 1 2024-03-29T04:58:01.138 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.db_write_buffer_size: 0 2024-03-29T04:58:01.138 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.write_buffer_manager: 0x555b05076820 2024-03-29T04:58:01.138 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-29T04:58:01.139 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-29T04:58:01.139 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-29T04:58:01.139 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.rate_limiter: (nil) 2024-03-29T04:58:01.139 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-29T04:58:01.139 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.wal_recovery_mode: 2 2024-03-29T04:58:01.139 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.enable_thread_tracking: 0 2024-03-29T04:58:01.139 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.enable_pipelined_write: 0 2024-03-29T04:58:01.139 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.unordered_write: 0 2024-03-29T04:58:01.139 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-29T04:58:01.139 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-29T04:58:01.139 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-29T04:58:01.139 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-29T04:58:01.139 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.row_cache: None 2024-03-29T04:58:01.139 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.wal_filter: None 2024-03-29T04:58:01.139 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-29T04:58:01.139 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.allow_ingest_behind: 0 2024-03-29T04:58:01.140 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.two_write_queues: 0 2024-03-29T04:58:01.140 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.manual_wal_flush: 0 2024-03-29T04:58:01.140 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.wal_compression: 0 2024-03-29T04:58:01.140 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.atomic_flush: 0 2024-03-29T04:58:01.140 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-29T04:58:01.140 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-29T04:58:01.140 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-29T04:58:01.140 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.log_readahead_size: 0 2024-03-29T04:58:01.140 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-29T04:58:01.140 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.best_efforts_recovery: 0 2024-03-29T04:58:01.140 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-29T04:58:01.140 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-29T04:58:01.140 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.allow_data_in_errors: 0 2024-03-29T04:58:01.140 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.db_host_id: __hostname__ 2024-03-29T04:58:01.140 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-29T04:58:01.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_background_jobs: 2 2024-03-29T04:58:01.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_background_compactions: -1 2024-03-29T04:58:01.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_subcompactions: 1 2024-03-29T04:58:01.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-29T04:58:01.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-29T04:58:01.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-29T04:58:01.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_total_wal_size: 0 2024-03-29T04:58:01.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-29T04:58:01.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-29T04:58:01.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-29T04:58:01.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-29T04:58:01.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_open_files: -1 2024-03-29T04:58:01.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.bytes_per_sync: 0 2024-03-29T04:58:01.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-29T04:58:01.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-29T04:58:01.142 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compaction_readahead_size: 0 2024-03-29T04:58:01.142 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_background_flushes: -1 2024-03-29T04:58:01.142 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Compression algorithms supported: 2024-03-29T04:58:01.142 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-29T04:58:01.142 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: kZSTD supported: 0 2024-03-29T04:58:01.142 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: kXpressCompression supported: 0 2024-03-29T04:58:01.142 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: kLZ4HCCompression supported: 1 2024-03-29T04:58:01.142 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: kZlibCompression supported: 1 2024-03-29T04:58:01.142 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: kSnappyCompression supported: 1 2024-03-29T04:58:01.142 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: kLZ4Compression supported: 1 2024-03-29T04:58:01.142 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: kBZip2Compression supported: 0 2024-03-29T04:58:01.142 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-29T04:58:01.142 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-29T04:58:01.142 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-03-29T04:58:01.143 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-29T04:58:01.143 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-29T04:58:01.143 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.merge_operator: 2024-03-29T04:58:01.143 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compaction_filter: None 2024-03-29T04:58:01.143 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compaction_filter_factory: None 2024-03-29T04:58:01.143 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.sst_partitioner_factory: None 2024-03-29T04:58:01.143 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-29T04:58:01.143 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-29T04:58:01.143 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555b050a3be0) 2024-03-29T04:58:01.143 INFO:journalctl@ceph.mon.b.smithi187.stdout: cache_index_and_filter_blocks: 1 2024-03-29T04:58:01.143 INFO:journalctl@ceph.mon.b.smithi187.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-29T04:58:01.143 INFO:journalctl@ceph.mon.b.smithi187.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-29T04:58:01.143 INFO:journalctl@ceph.mon.b.smithi187.stdout: pin_top_level_index_and_filter: 1 2024-03-29T04:58:01.143 INFO:journalctl@ceph.mon.b.smithi187.stdout: index_type: 0 2024-03-29T04:58:01.143 INFO:journalctl@ceph.mon.b.smithi187.stdout: data_block_index_type: 0 2024-03-29T04:58:01.144 INFO:journalctl@ceph.mon.b.smithi187.stdout: index_shortening: 1 2024-03-29T04:58:01.144 INFO:journalctl@ceph.mon.b.smithi187.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-29T04:58:01.144 INFO:journalctl@ceph.mon.b.smithi187.stdout: checksum: 4 2024-03-29T04:58:01.144 INFO:journalctl@ceph.mon.b.smithi187.stdout: no_block_cache: 0 2024-03-29T04:58:01.144 INFO:journalctl@ceph.mon.b.smithi187.stdout: block_cache: 0x555b050a7090 2024-03-29T04:58:01.144 INFO:journalctl@ceph.mon.b.smithi187.stdout: block_cache_name: BinnedLRUCache 2024-03-29T04:58:01.144 INFO:journalctl@ceph.mon.b.smithi187.stdout: block_cache_options: 2024-03-29T04:58:01.144 INFO:journalctl@ceph.mon.b.smithi187.stdout: capacity : 536870912 2024-03-29T04:58:01.144 INFO:journalctl@ceph.mon.b.smithi187.stdout: num_shard_bits : 4 2024-03-29T04:58:01.144 INFO:journalctl@ceph.mon.b.smithi187.stdout: strict_capacity_limit : 0 2024-03-29T04:58:01.144 INFO:journalctl@ceph.mon.b.smithi187.stdout: high_pri_pool_ratio: 0.000 2024-03-29T04:58:01.144 INFO:journalctl@ceph.mon.b.smithi187.stdout: block_cache_compressed: (nil) 2024-03-29T04:58:01.144 INFO:journalctl@ceph.mon.b.smithi187.stdout: persistent_cache: (nil) 2024-03-29T04:58:01.144 INFO:journalctl@ceph.mon.b.smithi187.stdout: block_size: 4096 2024-03-29T04:58:01.144 INFO:journalctl@ceph.mon.b.smithi187.stdout: block_size_deviation: 10 2024-03-29T04:58:01.145 INFO:journalctl@ceph.mon.b.smithi187.stdout: block_restart_interval: 16 2024-03-29T04:58:01.145 INFO:journalctl@ceph.mon.b.smithi187.stdout: index_block_restart_interval: 1 2024-03-29T04:58:01.145 INFO:journalctl@ceph.mon.b.smithi187.stdout: metadata_block_size: 4096 2024-03-29T04:58:01.145 INFO:journalctl@ceph.mon.b.smithi187.stdout: partition_filters: 0 2024-03-29T04:58:01.145 INFO:journalctl@ceph.mon.b.smithi187.stdout: use_delta_encoding: 1 2024-03-29T04:58:01.145 INFO:journalctl@ceph.mon.b.smithi187.stdout: filter_policy: bloomfilter 2024-03-29T04:58:01.145 INFO:journalctl@ceph.mon.b.smithi187.stdout: whole_key_filtering: 1 2024-03-29T04:58:01.145 INFO:journalctl@ceph.mon.b.smithi187.stdout: verify_compression: 0 2024-03-29T04:58:01.145 INFO:journalctl@ceph.mon.b.smithi187.stdout: read_amp_bytes_per_bit: 0 2024-03-29T04:58:01.145 INFO:journalctl@ceph.mon.b.smithi187.stdout: format_version: 5 2024-03-29T04:58:01.145 INFO:journalctl@ceph.mon.b.smithi187.stdout: enable_index_compression: 1 2024-03-29T04:58:01.145 INFO:journalctl@ceph.mon.b.smithi187.stdout: block_align: 0 2024-03-29T04:58:01.145 INFO:journalctl@ceph.mon.b.smithi187.stdout: max_auto_readahead_size: 262144 2024-03-29T04:58:01.145 INFO:journalctl@ceph.mon.b.smithi187.stdout: prepopulate_block_cache: 0 2024-03-29T04:58:01.145 INFO:journalctl@ceph.mon.b.smithi187.stdout: initial_auto_readahead_size: 8192 2024-03-29T04:58:01.146 INFO:journalctl@ceph.mon.b.smithi187.stdout: num_file_reads_for_auto_readahead: 2 2024-03-29T04:58:01.146 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.write_buffer_size: 33554432 2024-03-29T04:58:01.146 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_write_buffer_number: 2 2024-03-29T04:58:01.146 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compression: NoCompression 2024-03-29T04:58:01.146 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.bottommost_compression: Disabled 2024-03-29T04:58:01.146 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.prefix_extractor: nullptr 2024-03-29T04:58:01.146 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-29T04:58:01.146 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.num_levels: 7 2024-03-29T04:58:01.146 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-29T04:58:01.146 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-29T04:58:01.146 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-29T04:58:01.146 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-29T04:58:01.146 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-29T04:58:01.146 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-29T04:58:01.146 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-29T04:58:01.147 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:58:01.147 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-29T04:58:01.147 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-29T04:58:01.147 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:58:01.147 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:58:01.147 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-29T04:58:01.147 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compression_opts.level: 32767 2024-03-29T04:58:01.147 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compression_opts.strategy: 0 2024-03-29T04:58:01.147 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-29T04:58:01.147 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:58:01.147 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:58:01.147 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-29T04:58:01.147 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compression_opts.enabled: false 2024-03-29T04:58:01.147 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:58:01.147 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-29T04:58:01.148 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-29T04:58:01.148 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-29T04:58:01.148 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.target_file_size_base: 67108864 2024-03-29T04:58:01.148 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-29T04:58:01.148 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-29T04:58:01.148 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-29T04:58:01.148 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-29T04:58:01.148 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-29T04:58:01.148 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-29T04:58:01.148 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-29T04:58:01.148 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-29T04:58:01.148 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-29T04:58:01.148 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-29T04:58:01.148 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-29T04:58:01.148 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-29T04:58:01.149 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-29T04:58:01.149 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-29T04:58:01.149 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.arena_block_size: 1048576 2024-03-29T04:58:01.149 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-29T04:58:01.149 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-29T04:58:01.149 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.disable_auto_compactions: 0 2024-03-29T04:58:01.149 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-29T04:58:01.149 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-29T04:58:01.149 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-29T04:58:01.149 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-29T04:58:01.149 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-29T04:58:01.149 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-29T04:58:01.149 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-29T04:58:01.149 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-29T04:58:01.150 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-29T04:58:01.150 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-29T04:58:01.150 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.table_properties_collectors: 2024-03-29T04:58:01.150 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.inplace_update_support: 0 2024-03-29T04:58:01.150 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-29T04:58:01.150 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-29T04:58:01.150 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-29T04:58:01.150 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-29T04:58:01.150 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.bloom_locality: 0 2024-03-29T04:58:01.150 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.max_successive_merges: 0 2024-03-29T04:58:01.150 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-29T04:58:01.150 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.paranoid_file_checks: 0 2024-03-29T04:58:01.150 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.force_consistency_checks: 1 2024-03-29T04:58:01.150 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.report_bg_io_stats: 0 2024-03-29T04:58:01.150 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.ttl: 2592000 2024-03-29T04:58:01.151 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-29T04:58:01.151 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-29T04:58:01.151 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-29T04:58:01.151 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.enable_blob_files: false 2024-03-29T04:58:01.151 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.min_blob_size: 0 2024-03-29T04:58:01.151 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.blob_file_size: 268435456 2024-03-29T04:58:01.151 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-29T04:58:01.151 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-29T04:58:01.151 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-29T04:58:01.151 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-29T04:58:01.151 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-29T04:58:01.151 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.blob_file_starting_level: 0 2024-03-29T04:58:01.151 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-29T04:58:01.151 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-03-29T04:58:01.151 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:58:01.152 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13500, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-29T04:58:01.152 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-03-29T04:58:01.152 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 5488e726-2dc8-4369-8b0d-60725a8d0ece 2024-03-29T04:58:01.152 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711688280674069, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-03-29T04:58:01.152 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-03-29T04:58:01.152 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711688280703077, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 7078742, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13505, "largest_seqno": 18247, "table_properties": {"data_size": 7057604, "index_size": 14549, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 5573, "raw_key_size": 51471, "raw_average_key_size": 23, "raw_value_size": 7012248, "raw_average_value_size": 3181, "num_data_blocks": 622, "num_entries": 2204, "num_filter_entries": 2204, "num_deletions": 257, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1711688280, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "5488e726-2dc8-4369-8b0d-60725a8d0ece", "db_session_id": "GP2DFNGQVQUS41VD359A", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-03-29T04:58:01.152 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711688280703163, "job": 1, "event": "recovery_finished"} 2024-03-29T04:58:01.152 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-03-29T04:58:01.152 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:58:01.152 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-29T04:58:01.152 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x555b05198000 2024-03-29T04:58:01.152 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: DB pointer 0x555b05182000 2024-03-29T04:58:01.152 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-29T04:58:01.152 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-29T04:58:01.153 INFO:journalctl@ceph.mon.b.smithi187.stdout: ** DB Stats ** 2024-03-29T04:58:01.153 INFO:journalctl@ceph.mon.b.smithi187.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:58:01.153 INFO:journalctl@ceph.mon.b.smithi187.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-29T04:58:01.153 INFO:journalctl@ceph.mon.b.smithi187.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:58:01.153 INFO:journalctl@ceph.mon.b.smithi187.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:58:01.153 INFO:journalctl@ceph.mon.b.smithi187.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-29T04:58:01.153 INFO:journalctl@ceph.mon.b.smithi187.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:58:01.153 INFO:journalctl@ceph.mon.b.smithi187.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:58:01.153 INFO:journalctl@ceph.mon.b.smithi187.stdout: 2024-03-29T04:58:01.153 INFO:journalctl@ceph.mon.b.smithi187.stdout: ** Compaction Stats [default] ** 2024-03-29T04:58:01.153 INFO:journalctl@ceph.mon.b.smithi187.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-29T04:58:01.153 INFO:journalctl@ceph.mon.b.smithi187.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-29T04:58:01.153 INFO:journalctl@ceph.mon.b.smithi187.stdout: L0 1/0 6.75 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 594.9 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-03-29T04:58:01.153 INFO:journalctl@ceph.mon.b.smithi187.stdout: L6 1/0 8.35 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-29T04:58:01.154 INFO:journalctl@ceph.mon.b.smithi187.stdout: Sum 2/0 15.10 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 594.9 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-03-29T04:58:01.154 INFO:journalctl@ceph.mon.b.smithi187.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 594.9 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-03-29T04:58:01.154 INFO:journalctl@ceph.mon.b.smithi187.stdout: 2024-03-29T04:58:01.154 INFO:journalctl@ceph.mon.b.smithi187.stdout: ** Compaction Stats [default] ** 2024-03-29T04:58:01.154 INFO:journalctl@ceph.mon.b.smithi187.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-29T04:58:01.154 INFO:journalctl@ceph.mon.b.smithi187.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:58:01.154 INFO:journalctl@ceph.mon.b.smithi187.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 594.9 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-03-29T04:58:01.154 INFO:journalctl@ceph.mon.b.smithi187.stdout: 2024-03-29T04:58:01.154 INFO:journalctl@ceph.mon.b.smithi187.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-29T04:58:01.154 INFO:journalctl@ceph.mon.b.smithi187.stdout: 2024-03-29T04:58:01.154 INFO:journalctl@ceph.mon.b.smithi187.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:58:01.154 INFO:journalctl@ceph.mon.b.smithi187.stdout: Flush(GB): cumulative 0.007, interval 0.007 2024-03-29T04:58:01.154 INFO:journalctl@ceph.mon.b.smithi187.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:58:01.155 INFO:journalctl@ceph.mon.b.smithi187.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:58:01.155 INFO:journalctl@ceph.mon.b.smithi187.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:58:01.155 INFO:journalctl@ceph.mon.b.smithi187.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:58:01.155 INFO:journalctl@ceph.mon.b.smithi187.stdout: Cumulative compaction: 0.01 GB write, 202.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:58:01.155 INFO:journalctl@ceph.mon.b.smithi187.stdout: Interval compaction: 0.01 GB write, 202.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:58:01.155 INFO:journalctl@ceph.mon.b.smithi187.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:58:01.155 INFO:journalctl@ceph.mon.b.smithi187.stdout: Block cache BinnedLRUCache@0x555b050a7090#7 capacity: 512.00 MB usage: 22.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-03-29T04:58:01.155 INFO:journalctl@ceph.mon.b.smithi187.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,6.03 KB,0.00115037%) IndexBlock(1,16.08 KB,0.00306666%) Misc(1,0.00 KB,0%) 2024-03-29T04:58:01.155 INFO:journalctl@ceph.mon.b.smithi187.stdout: 2024-03-29T04:58:01.155 INFO:journalctl@ceph.mon.b.smithi187.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:58:01.155 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: starting mon.b rank 1 at public addrs [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] at bind addrs [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:58:01.155 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: mon.b@-1(???) e3 preinit fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:58:01.155 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: mon.b@-1(???).mds e9 new map 2024-03-29T04:58:01.156 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: mon.b@-1(???).mds e9 print_map 2024-03-29T04:58:01.156 INFO:journalctl@ceph.mon.b.smithi187.stdout: e9 2024-03-29T04:58:01.156 INFO:journalctl@ceph.mon.b.smithi187.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-29T04:58:01.156 INFO:journalctl@ceph.mon.b.smithi187.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:58:01.156 INFO:journalctl@ceph.mon.b.smithi187.stdout: legacy client fscid: 1 2024-03-29T04:58:01.156 INFO:journalctl@ceph.mon.b.smithi187.stdout: 2024-03-29T04:58:01.156 INFO:journalctl@ceph.mon.b.smithi187.stdout: Filesystem 'foo' (1) 2024-03-29T04:58:01.156 INFO:journalctl@ceph.mon.b.smithi187.stdout: fs_name foo 2024-03-29T04:58:01.156 INFO:journalctl@ceph.mon.b.smithi187.stdout: epoch 8 2024-03-29T04:58:01.156 INFO:journalctl@ceph.mon.b.smithi187.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-29T04:58:01.156 INFO:journalctl@ceph.mon.b.smithi187.stdout: created 2024-03-29T04:51:50.886956+0000 2024-03-29T04:58:01.156 INFO:journalctl@ceph.mon.b.smithi187.stdout: modified 2024-03-29T04:52:07.243441+0000 2024-03-29T04:58:01.156 INFO:journalctl@ceph.mon.b.smithi187.stdout: tableserver 0 2024-03-29T04:58:01.156 INFO:journalctl@ceph.mon.b.smithi187.stdout: root 0 2024-03-29T04:58:01.157 INFO:journalctl@ceph.mon.b.smithi187.stdout: session_timeout 60 2024-03-29T04:58:01.157 INFO:journalctl@ceph.mon.b.smithi187.stdout: session_autoclose 300 2024-03-29T04:58:01.157 INFO:journalctl@ceph.mon.b.smithi187.stdout: max_file_size 1099511627776 2024-03-29T04:58:01.157 INFO:journalctl@ceph.mon.b.smithi187.stdout: max_xattr_size 65536 2024-03-29T04:58:01.157 INFO:journalctl@ceph.mon.b.smithi187.stdout: required_client_features {} 2024-03-29T04:58:01.157 INFO:journalctl@ceph.mon.b.smithi187.stdout: last_failure 0 2024-03-29T04:58:01.157 INFO:journalctl@ceph.mon.b.smithi187.stdout: last_failure_osd_epoch 0 2024-03-29T04:58:01.157 INFO:journalctl@ceph.mon.b.smithi187.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:58:01.157 INFO:journalctl@ceph.mon.b.smithi187.stdout: max_mds 1 2024-03-29T04:58:01.157 INFO:journalctl@ceph.mon.b.smithi187.stdout: in 0 2024-03-29T04:58:01.157 INFO:journalctl@ceph.mon.b.smithi187.stdout: up {0=24560} 2024-03-29T04:58:01.157 INFO:journalctl@ceph.mon.b.smithi187.stdout: failed 2024-03-29T04:58:01.157 INFO:journalctl@ceph.mon.b.smithi187.stdout: damaged 2024-03-29T04:58:01.157 INFO:journalctl@ceph.mon.b.smithi187.stdout: stopped 2024-03-29T04:58:01.157 INFO:journalctl@ceph.mon.b.smithi187.stdout: data_pools [4] 2024-03-29T04:58:01.158 INFO:journalctl@ceph.mon.b.smithi187.stdout: metadata_pool 3 2024-03-29T04:58:01.158 INFO:journalctl@ceph.mon.b.smithi187.stdout: inline_data disabled 2024-03-29T04:58:01.158 INFO:journalctl@ceph.mon.b.smithi187.stdout: balancer 2024-03-29T04:58:01.158 INFO:journalctl@ceph.mon.b.smithi187.stdout: bal_rank_mask -1 2024-03-29T04:58:01.158 INFO:journalctl@ceph.mon.b.smithi187.stdout: standby_count_wanted 1 2024-03-29T04:58:01.158 INFO:journalctl@ceph.mon.b.smithi187.stdout: qdb_cluster leader: 0 members: 2024-03-29T04:58:01.158 INFO:journalctl@ceph.mon.b.smithi187.stdout: [mds.foo.smithi181.pflrwm{0:24560} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.181:6834/2778467308,v1:172.21.15.181:6835/2778467308] compat {c=[1],r=[1],i=[7ff]}] 2024-03-29T04:58:01.158 INFO:journalctl@ceph.mon.b.smithi187.stdout: 2024-03-29T04:58:01.158 INFO:journalctl@ceph.mon.b.smithi187.stdout: 2024-03-29T04:58:01.158 INFO:journalctl@ceph.mon.b.smithi187.stdout: Standby daemons: 2024-03-29T04:58:01.158 INFO:journalctl@ceph.mon.b.smithi187.stdout: 2024-03-29T04:58:01.158 INFO:journalctl@ceph.mon.b.smithi187.stdout: [mds.foo.smithi187.vcsnqx{-1:24511} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.187:6832/1383464008,v1:172.21.15.187:6833/1383464008] compat {c=[1],r=[1],i=[7ff]}] 2024-03-29T04:58:01.158 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: mon.b@-1(???).osd e342 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-29T04:58:01.158 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: mon.b@-1(???).osd e342 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:58:01.158 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: mon.b@-1(???).osd e342 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:58:01.159 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: mon.b@-1(???).osd e342 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:58:01.159 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 ceph-mon[93536]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-03-29T04:58:01.159 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 bash[93498]: 4de0ae3c44c36f2dc00ac3f31d42bb83b5867282720d6b8baf7aa7fb7949eea5 2024-03-29T04:58:01.159 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:00 smithi187 systemd[1]: Started Ceph mon.b for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:58:02.895 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 175104738 2024-03-29T04:58:02.938 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 17510473~4377618 2024-03-29T04:58:02.942 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 18604877~2188809 2024-03-29T04:58:02.982 INFO:tasks.workunit.client.0.smithi181.stdout:write: 2 35020947~4377618 2024-03-29T04:58:02.984 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 36115351~2188809 2024-03-29T04:58:03.042 INFO:tasks.workunit.client.0.smithi181.stdout:write: 3 52531421~4377618 2024-03-29T04:58:03.043 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 53625825~2188809 2024-03-29T04:58:03.080 INFO:tasks.workunit.client.0.smithi181.stdout:write: 4 70041895~4377618 2024-03-29T04:58:03.084 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 71136299~2188809 2024-03-29T04:58:03.148 INFO:tasks.workunit.client.0.smithi181.stdout:write: 5 87552369~4377618 2024-03-29T04:58:03.149 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 88646773~2188809 2024-03-29T04:58:03.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: mon.b calling monitor election 2024-03-29T04:58:03.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: mon.c calling monitor election 2024-03-29T04:58:03.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:58:03.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:58:03.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:58:03.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: mon.a calling monitor election 2024-03-29T04:58:03.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:58:03.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: monmap epoch 4 2024-03-29T04:58:03.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:58:03.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: last_changed 2024-03-29T04:58:01.863831+0000 2024-03-29T04:58:03.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: created 2024-03-29T04:44:46.123593+0000 2024-03-29T04:58:03.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: min_mon_release 19 (squid) 2024-03-29T04:58:03.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: election_strategy: 1 2024-03-29T04:58:03.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: 0: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.a 2024-03-29T04:58:03.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: 1: [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon.b 2024-03-29T04:58:03.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: 2: [v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0] mon.c 2024-03-29T04:58:03.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:58:03.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: osdmap e342: 8 total, 8 up, 8 in 2024-03-29T04:58:03.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: mgrmap e37: y(active, since 110s), standbys: x 2024-03-29T04:58:03.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-03-29T04:58:03.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:58:03.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: daemon grafana.a on smithi187 is in error state 2024-03-29T04:58:03.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-29T04:58:03.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: application not enabled on pool 'test-librbd-smithi181-86707-9' 2024-03-29T04:58:03.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-29T04:58:03.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:03.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:03.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:58:03.209 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: mon.b calling monitor election 2024-03-29T04:58:03.209 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: mon.c calling monitor election 2024-03-29T04:58:03.209 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:58:03.209 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:58:03.209 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:58:03.209 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: mon.a calling monitor election 2024-03-29T04:58:03.210 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:58:03.210 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: monmap epoch 4 2024-03-29T04:58:03.210 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:58:03.210 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: last_changed 2024-03-29T04:58:01.863831+0000 2024-03-29T04:58:03.210 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: created 2024-03-29T04:44:46.123593+0000 2024-03-29T04:58:03.210 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: min_mon_release 19 (squid) 2024-03-29T04:58:03.210 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: election_strategy: 1 2024-03-29T04:58:03.210 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: 0: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.a 2024-03-29T04:58:03.210 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: 1: [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon.b 2024-03-29T04:58:03.211 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: 2: [v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0] mon.c 2024-03-29T04:58:03.211 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:58:03.211 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: osdmap e342: 8 total, 8 up, 8 in 2024-03-29T04:58:03.211 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: mgrmap e37: y(active, since 110s), standbys: x 2024-03-29T04:58:03.211 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-03-29T04:58:03.211 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:58:03.211 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: daemon grafana.a on smithi187 is in error state 2024-03-29T04:58:03.211 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-29T04:58:03.211 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: application not enabled on pool 'test-librbd-smithi181-86707-9' 2024-03-29T04:58:03.212 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-29T04:58:03.212 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:03.212 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:03.212 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:02 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:58:03.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: mon.b calling monitor election 2024-03-29T04:58:03.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: mon.c calling monitor election 2024-03-29T04:58:03.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:58:03.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:58:03.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:58:03.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: mon.a calling monitor election 2024-03-29T04:58:03.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:58:03.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: monmap epoch 4 2024-03-29T04:58:03.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: fsid dfcecab8-ed86-11ee-95d0-87774f69a715 2024-03-29T04:58:03.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: last_changed 2024-03-29T04:58:01.863831+0000 2024-03-29T04:58:03.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: created 2024-03-29T04:44:46.123593+0000 2024-03-29T04:58:03.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: min_mon_release 19 (squid) 2024-03-29T04:58:03.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: election_strategy: 1 2024-03-29T04:58:03.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: 0: [v2:172.21.15.181:3300/0,v1:172.21.15.181:6789/0] mon.a 2024-03-29T04:58:03.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: 1: [v2:172.21.15.187:3300/0,v1:172.21.15.187:6789/0] mon.b 2024-03-29T04:58:03.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: 2: [v2:172.21.15.181:3301/0,v1:172.21.15.181:6790/0] mon.c 2024-03-29T04:58:03.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: fsmap foo:1 {0=foo.smithi181.pflrwm=up:active} 1 up:standby 2024-03-29T04:58:03.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: osdmap e342: 8 total, 8 up, 8 in 2024-03-29T04:58:03.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: mgrmap e37: y(active, since 110s), standbys: x 2024-03-29T04:58:03.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); 1 pool(s) do not have an application enabled 2024-03-29T04:58:03.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:58:03.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: daemon grafana.a on smithi187 is in error state 2024-03-29T04:58:03.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-29T04:58:03.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: application not enabled on pool 'test-librbd-smithi181-86707-9' 2024-03-29T04:58:03.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-29T04:58:03.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:03.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:03.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:02 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:58:04.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:03 smithi187 ceph-mon[93536]: pgmap v161: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 8.4 MiB/s rd, 33 MiB/s wr, 307 op/s 2024-03-29T04:58:04.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:03 smithi187 ceph-mon[93536]: failed to encode map e343 with expected crc 2024-03-29T04:58:04.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:03 smithi187 ceph-mon[93536]: failed to encode map e343 with expected crc 2024-03-29T04:58:04.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:03 smithi187 ceph-mon[93536]: osdmap e343: 8 total, 8 up, 8 in 2024-03-29T04:58:04.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:03 smithi187 ceph-mon[93536]: failed to encode map e343 with expected crc 2024-03-29T04:58:04.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:03 smithi181 ceph-mon[101970]: pgmap v161: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 8.4 MiB/s rd, 33 MiB/s wr, 307 op/s 2024-03-29T04:58:04.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:03 smithi181 ceph-mon[101970]: failed to encode map e343 with expected crc 2024-03-29T04:58:04.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:03 smithi181 ceph-mon[101970]: failed to encode map e343 with expected crc 2024-03-29T04:58:04.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:03 smithi181 ceph-mon[101970]: osdmap e343: 8 total, 8 up, 8 in 2024-03-29T04:58:04.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:03 smithi181 ceph-mon[101970]: failed to encode map e343 with expected crc 2024-03-29T04:58:04.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:03 smithi181 ceph-mon[106466]: pgmap v161: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 8.4 MiB/s rd, 33 MiB/s wr, 307 op/s 2024-03-29T04:58:04.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:03 smithi181 ceph-mon[106466]: failed to encode map e343 with expected crc 2024-03-29T04:58:04.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:03 smithi181 ceph-mon[106466]: failed to encode map e343 with expected crc 2024-03-29T04:58:04.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:03 smithi181 ceph-mon[106466]: osdmap e343: 8 total, 8 up, 8 in 2024-03-29T04:58:04.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:03 smithi181 ceph-mon[106466]: failed to encode map e343 with expected crc 2024-03-29T04:58:04.818 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:58:05.062 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:58:06.111 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:05 smithi187 ceph-mon[93536]: pgmap v163: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 7.3 MiB/s rd, 29 MiB/s wr, 268 op/s 2024-03-29T04:58:06.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:05 smithi181 ceph-mon[101970]: pgmap v163: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 7.3 MiB/s rd, 29 MiB/s wr, 268 op/s 2024-03-29T04:58:06.203 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:58:05 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T04:58:05.713+0000 7f67753a3700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-03-29T04:58:06.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:05 smithi181 ceph-mon[106466]: pgmap v163: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 7.3 MiB/s rd, 29 MiB/s wr, 268 op/s 2024-03-29T04:58:06.759 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.Stress2 (32511 ms) 2024-03-29T04:58:06.759 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMigration.StressLive 2024-03-29T04:58:06.792 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 83886080 2024-03-29T04:58:06.802 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 77183158~6702922 2024-03-29T04:58:06.847 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 9022895~5247576 2024-03-29T04:58:06.847 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 19715857~371564 2024-03-29T04:58:06.854 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 53111920~6617793 2024-03-29T04:58:06.854 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 48048133~694240 2024-03-29T04:58:06.863 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 17904862~430219 2024-03-29T04:58:06.865 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 76436545~588089 2024-03-29T04:58:06.872 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 6611714~3731231 2024-03-29T04:58:06.872 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap0 2024-03-29T04:58:07.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:07 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:07.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:07 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:07.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:07 smithi187 ceph-mon[93536]: pgmap v164: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 1.1 KiB/s rd, 464 B/s wr, 1 op/s 2024-03-29T04:58:07.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:07 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:07.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:07 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:07.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:07 smithi181 ceph-mon[106466]: pgmap v164: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 1.1 KiB/s rd, 464 B/s wr, 1 op/s 2024-03-29T04:58:07.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:07 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:07.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:07 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:07.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:07 smithi181 ceph-mon[101970]: pgmap v164: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 1.1 KiB/s rd, 464 B/s wr, 1 op/s 2024-03-29T04:58:08.493 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap0 2024-03-29T04:58:08.504 INFO:tasks.workunit.client.0.smithi181.stdout:clone image185 -> image188 2024-03-29T04:58:08.595 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 70786008~4162970 2024-03-29T04:58:08.637 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 6413397~2044364 2024-03-29T04:58:08.643 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 5780356~2297592 2024-03-29T04:58:08.657 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 75889998~678006 2024-03-29T04:58:08.664 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:08 smithi187 ceph-mon[93536]: failed to encode map e344 with expected crc 2024-03-29T04:58:08.664 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:08 smithi187 ceph-mon[93536]: failed to encode map e344 with expected crc 2024-03-29T04:58:08.664 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:08 smithi187 ceph-mon[93536]: osdmap e344: 8 total, 8 up, 8 in 2024-03-29T04:58:08.664 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:08 smithi187 ceph-mon[93536]: failed to encode map e344 with expected crc 2024-03-29T04:58:08.664 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:08 smithi187 ceph-mon[93536]: failed to encode map e344 with expected crc 2024-03-29T04:58:08.664 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:08 smithi187 ceph-mon[93536]: failed to encode map e344 with expected crc 2024-03-29T04:58:08.664 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:08 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:08.664 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:08 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:08.692 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 79590902~4295178 2024-03-29T04:58:08.724 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 66197049~7433186 2024-03-29T04:58:08.729 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 22524716~7963215 2024-03-29T04:58:08.782 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 29988007~3456436 2024-03-29T04:58:08.784 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap1 2024-03-29T04:58:08.854 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-mon[106466]: failed to encode map e344 with expected crc 2024-03-29T04:58:08.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-mon[106466]: failed to encode map e344 with expected crc 2024-03-29T04:58:08.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-mon[106466]: osdmap e344: 8 total, 8 up, 8 in 2024-03-29T04:58:08.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-mon[106466]: failed to encode map e344 with expected crc 2024-03-29T04:58:08.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-mon[106466]: failed to encode map e344 with expected crc 2024-03-29T04:58:08.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-mon[106466]: failed to encode map e344 with expected crc 2024-03-29T04:58:08.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:08.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:08.855 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-mon[101970]: failed to encode map e344 with expected crc 2024-03-29T04:58:08.855 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-mon[101970]: failed to encode map e344 with expected crc 2024-03-29T04:58:08.855 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-mon[101970]: osdmap e344: 8 total, 8 up, 8 in 2024-03-29T04:58:08.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-mon[101970]: failed to encode map e344 with expected crc 2024-03-29T04:58:08.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-mon[101970]: failed to encode map e344 with expected crc 2024-03-29T04:58:08.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-mon[101970]: failed to encode map e344 with expected crc 2024-03-29T04:58:08.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:08.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:09.004 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:58:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:58:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:58:09.203 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:58:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:58:08] "GET /metrics HTTP/1.1" 200 38721 "" "Prometheus/2.43.0" 2024-03-29T04:58:09.691 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:09 smithi187 ceph-mon[93536]: failed to encode map e345 with expected crc 2024-03-29T04:58:09.692 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:09 smithi187 ceph-mon[93536]: failed to encode map e345 with expected crc 2024-03-29T04:58:09.692 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:09 smithi187 ceph-mon[93536]: osdmap e345: 8 total, 8 up, 8 in 2024-03-29T04:58:09.692 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:09 smithi187 ceph-mon[93536]: failed to encode map e345 with expected crc 2024-03-29T04:58:09.692 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:09 smithi187 ceph-mon[93536]: pgmap v167: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 322 MiB/s rd, 13 MiB/s wr, 713 op/s 2024-03-29T04:58:09.874 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:09 smithi181 ceph-mon[101970]: failed to encode map e345 with expected crc 2024-03-29T04:58:09.874 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:09 smithi181 ceph-mon[101970]: failed to encode map e345 with expected crc 2024-03-29T04:58:09.874 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:09 smithi181 ceph-mon[101970]: osdmap e345: 8 total, 8 up, 8 in 2024-03-29T04:58:09.874 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:09 smithi181 ceph-mon[101970]: failed to encode map e345 with expected crc 2024-03-29T04:58:09.874 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:09 smithi181 ceph-mon[101970]: pgmap v167: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 322 MiB/s rd, 13 MiB/s wr, 713 op/s 2024-03-29T04:58:09.875 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:09 smithi181 ceph-mon[106466]: failed to encode map e345 with expected crc 2024-03-29T04:58:09.875 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:09 smithi181 ceph-mon[106466]: failed to encode map e345 with expected crc 2024-03-29T04:58:09.875 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:09 smithi181 ceph-mon[106466]: osdmap e345: 8 total, 8 up, 8 in 2024-03-29T04:58:09.875 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:09 smithi181 ceph-mon[106466]: failed to encode map e345 with expected crc 2024-03-29T04:58:09.875 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:09 smithi181 ceph-mon[106466]: pgmap v167: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 322 MiB/s rd, 13 MiB/s wr, 713 op/s 2024-03-29T04:58:10.504 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap1 2024-03-29T04:58:10.513 INFO:tasks.workunit.client.0.smithi181.stdout:clone image188 -> image190 2024-03-29T04:58:10.635 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 54942621~7760824 2024-03-29T04:58:10.696 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 33620888~6608272 2024-03-29T04:58:10.704 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 40592924~2327458 2024-03-29T04:58:10.719 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 36469980~1194103 2024-03-29T04:58:10.721 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 38479766~7977565 2024-03-29T04:58:10.728 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:10 smithi187 ceph-mon[93536]: failed to encode map e346 with expected crc 2024-03-29T04:58:10.728 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:10 smithi187 ceph-mon[93536]: failed to encode map e346 with expected crc 2024-03-29T04:58:10.729 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:10 smithi187 ceph-mon[93536]: osdmap e346: 8 total, 8 up, 8 in 2024-03-29T04:58:10.729 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:10 smithi187 ceph-mon[93536]: failed to encode map e346 with expected crc 2024-03-29T04:58:10.790 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 70380742~1138654 2024-03-29T04:58:10.808 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 64021880~7742094 2024-03-29T04:58:10.850 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 26563675~6949023 2024-03-29T04:58:10.881 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap2 2024-03-29T04:58:10.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:10 smithi181 ceph-mon[106466]: failed to encode map e346 with expected crc 2024-03-29T04:58:10.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:10 smithi181 ceph-mon[106466]: failed to encode map e346 with expected crc 2024-03-29T04:58:10.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:10 smithi181 ceph-mon[106466]: osdmap e346: 8 total, 8 up, 8 in 2024-03-29T04:58:10.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:10 smithi181 ceph-mon[106466]: failed to encode map e346 with expected crc 2024-03-29T04:58:10.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:10 smithi181 ceph-mon[101970]: failed to encode map e346 with expected crc 2024-03-29T04:58:10.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:10 smithi181 ceph-mon[101970]: failed to encode map e346 with expected crc 2024-03-29T04:58:10.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:10 smithi181 ceph-mon[101970]: osdmap e346: 8 total, 8 up, 8 in 2024-03-29T04:58:10.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:10 smithi181 ceph-mon[101970]: failed to encode map e346 with expected crc 2024-03-29T04:58:11.846 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[106466]: failed to encode map e347 with expected crc 2024-03-29T04:58:11.846 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[106466]: failed to encode map e347 with expected crc 2024-03-29T04:58:11.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[106466]: osdmap e347: 8 total, 8 up, 8 in 2024-03-29T04:58:11.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[106466]: failed to encode map e347 with expected crc 2024-03-29T04:58:11.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[106466]: failed to encode map e347 with expected crc 2024-03-29T04:58:11.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[106466]: pgmap v170: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 484 MiB/s rd, 20 MiB/s wr, 1.07k op/s 2024-03-29T04:58:11.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:11.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:11.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:11.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:58:11.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:11.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:11.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:58:11.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:58:11.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:11.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[101970]: failed to encode map e347 with expected crc 2024-03-29T04:58:11.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[101970]: failed to encode map e347 with expected crc 2024-03-29T04:58:11.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[101970]: osdmap e347: 8 total, 8 up, 8 in 2024-03-29T04:58:11.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[101970]: failed to encode map e347 with expected crc 2024-03-29T04:58:11.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[101970]: failed to encode map e347 with expected crc 2024-03-29T04:58:11.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[101970]: pgmap v170: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 484 MiB/s rd, 20 MiB/s wr, 1.07k op/s 2024-03-29T04:58:11.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:11.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:11.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:11.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:58:11.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:11.849 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:11.849 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:58:11.849 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:58:11.849 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:11.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:11 smithi187 ceph-mon[93536]: failed to encode map e347 with expected crc 2024-03-29T04:58:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:11 smithi187 ceph-mon[93536]: failed to encode map e347 with expected crc 2024-03-29T04:58:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:11 smithi187 ceph-mon[93536]: osdmap e347: 8 total, 8 up, 8 in 2024-03-29T04:58:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:11 smithi187 ceph-mon[93536]: failed to encode map e347 with expected crc 2024-03-29T04:58:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:11 smithi187 ceph-mon[93536]: failed to encode map e347 with expected crc 2024-03-29T04:58:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:11 smithi187 ceph-mon[93536]: pgmap v170: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 484 MiB/s rd, 20 MiB/s wr, 1.07k op/s 2024-03-29T04:58:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:58:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:58:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:58:11.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:12.520 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap2 2024-03-29T04:58:12.530 INFO:tasks.workunit.client.0.smithi181.stdout:clone image190 -> image192 2024-03-29T04:58:12.674 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 149784543 2024-03-29T04:58:12.685 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 109176235~4004420 2024-03-29T04:58:12.735 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 539985~3084778 2024-03-29T04:58:12.741 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 50168008~295111 2024-03-29T04:58:12.788 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 124777723~1156595 2024-03-29T04:58:12.789 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 74298044~5629399 2024-03-29T04:58:12.860 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 55115842~4035791 2024-03-29T04:58:12.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:12 smithi187 ceph-mon[93536]: failed to encode map e347 with expected crc 2024-03-29T04:58:12.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:12 smithi187 ceph-mon[93536]: pgmap v171: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-03-29T04:58:12.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:12 smithi187 ceph-mon[93536]: pgmap v172: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-03-29T04:58:12.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:12 smithi187 ceph-mon[93536]: Reconfiguring mon.a (monmap changed)... 2024-03-29T04:58:12.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:12 smithi187 ceph-mon[93536]: Reconfiguring daemon mon.a on smithi181 2024-03-29T04:58:12.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:12 smithi187 ceph-mon[93536]: failed to encode map e348 with expected crc 2024-03-29T04:58:12.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:12 smithi187 ceph-mon[93536]: failed to encode map e348 with expected crc 2024-03-29T04:58:12.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:12 smithi187 ceph-mon[93536]: osdmap e348: 8 total, 8 up, 8 in 2024-03-29T04:58:12.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:12 smithi187 ceph-mon[93536]: failed to encode map e348 with expected crc 2024-03-29T04:58:12.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:12 smithi187 ceph-mon[93536]: failed to encode map e348 with expected crc 2024-03-29T04:58:12.910 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 11151765~5350995 2024-03-29T04:58:12.946 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 89303532~7677064 2024-03-29T04:58:12.946 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: snap3 2024-03-29T04:58:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[106466]: failed to encode map e347 with expected crc 2024-03-29T04:58:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[106466]: pgmap v171: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-03-29T04:58:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[106466]: pgmap v172: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-03-29T04:58:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[106466]: Reconfiguring mon.a (monmap changed)... 2024-03-29T04:58:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[106466]: Reconfiguring daemon mon.a on smithi181 2024-03-29T04:58:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[106466]: failed to encode map e348 with expected crc 2024-03-29T04:58:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[106466]: failed to encode map e348 with expected crc 2024-03-29T04:58:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[106466]: osdmap e348: 8 total, 8 up, 8 in 2024-03-29T04:58:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[106466]: failed to encode map e348 with expected crc 2024-03-29T04:58:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[106466]: failed to encode map e348 with expected crc 2024-03-29T04:58:12.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[101970]: failed to encode map e347 with expected crc 2024-03-29T04:58:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[101970]: pgmap v171: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-03-29T04:58:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[101970]: pgmap v172: 137 pgs: 137 active+clean; 1.9 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-03-29T04:58:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[101970]: Reconfiguring mon.a (monmap changed)... 2024-03-29T04:58:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[101970]: Reconfiguring daemon mon.a on smithi181 2024-03-29T04:58:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[101970]: failed to encode map e348 with expected crc 2024-03-29T04:58:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[101970]: failed to encode map e348 with expected crc 2024-03-29T04:58:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[101970]: osdmap e348: 8 total, 8 up, 8 in 2024-03-29T04:58:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[101970]: failed to encode map e348 with expected crc 2024-03-29T04:58:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:12 smithi181 ceph-mon[101970]: failed to encode map e348 with expected crc 2024-03-29T04:58:13.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[106466]: failed to encode map e349 with expected crc 2024-03-29T04:58:13.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[106466]: failed to encode map e349 with expected crc 2024-03-29T04:58:13.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[106466]: osdmap e349: 8 total, 8 up, 8 in 2024-03-29T04:58:13.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[106466]: failed to encode map e349 with expected crc 2024-03-29T04:58:13.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[106466]: failed to encode map e349 with expected crc 2024-03-29T04:58:13.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:13.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:13.847 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:58:13.848 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:58:13.848 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:13.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[101970]: failed to encode map e349 with expected crc 2024-03-29T04:58:13.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[101970]: failed to encode map e349 with expected crc 2024-03-29T04:58:13.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[101970]: osdmap e349: 8 total, 8 up, 8 in 2024-03-29T04:58:13.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[101970]: failed to encode map e349 with expected crc 2024-03-29T04:58:13.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[101970]: failed to encode map e349 with expected crc 2024-03-29T04:58:13.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:13.848 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:13.849 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:58:13.849 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:58:13.849 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:13.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:13 smithi187 ceph-mon[93536]: failed to encode map e349 with expected crc 2024-03-29T04:58:13.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:13 smithi187 ceph-mon[93536]: failed to encode map e349 with expected crc 2024-03-29T04:58:13.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:13 smithi187 ceph-mon[93536]: osdmap e349: 8 total, 8 up, 8 in 2024-03-29T04:58:13.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:13 smithi187 ceph-mon[93536]: failed to encode map e349 with expected crc 2024-03-29T04:58:13.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:13 smithi187 ceph-mon[93536]: failed to encode map e349 with expected crc 2024-03-29T04:58:13.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:13.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:13.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:58:13.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:58:13.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:14.525 INFO:tasks.workunit.client.0.smithi181.stdout:snap_protect: snap3 2024-03-29T04:58:14.536 INFO:tasks.workunit.client.0.smithi181.stdout:clone image192 -> image194 2024-03-29T04:58:14.713 INFO:tasks.workunit.client.0.smithi181.stdout:migration_prepare 2024-03-29T04:58:14.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:14 smithi187 ceph-mon[93536]: pgmap v175: 137 pgs: 137 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 7.5 MiB/s rd, 39 MiB/s wr, 590 op/s 2024-03-29T04:58:14.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:14 smithi187 ceph-mon[93536]: Reconfiguring mgr.y (monmap changed)... 2024-03-29T04:58:14.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:14 smithi187 ceph-mon[93536]: Reconfiguring daemon mgr.y on smithi181 2024-03-29T04:58:14.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:14 smithi187 ceph-mon[93536]: failed to encode map e350 with expected crc 2024-03-29T04:58:14.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:14 smithi187 ceph-mon[93536]: failed to encode map e350 with expected crc 2024-03-29T04:58:14.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:14 smithi187 ceph-mon[93536]: osdmap e350: 8 total, 8 up, 8 in 2024-03-29T04:58:14.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:14 smithi187 ceph-mon[93536]: failed to encode map e350 with expected crc 2024-03-29T04:58:14.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:14 smithi181 ceph-mon[106466]: pgmap v175: 137 pgs: 137 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 7.5 MiB/s rd, 39 MiB/s wr, 590 op/s 2024-03-29T04:58:14.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:14 smithi181 ceph-mon[106466]: Reconfiguring mgr.y (monmap changed)... 2024-03-29T04:58:14.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:14 smithi181 ceph-mon[106466]: Reconfiguring daemon mgr.y on smithi181 2024-03-29T04:58:14.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:14 smithi181 ceph-mon[106466]: failed to encode map e350 with expected crc 2024-03-29T04:58:14.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:14 smithi181 ceph-mon[106466]: failed to encode map e350 with expected crc 2024-03-29T04:58:14.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:14 smithi181 ceph-mon[106466]: osdmap e350: 8 total, 8 up, 8 in 2024-03-29T04:58:14.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:14 smithi181 ceph-mon[106466]: failed to encode map e350 with expected crc 2024-03-29T04:58:14.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:14 smithi181 ceph-mon[101970]: pgmap v175: 137 pgs: 137 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 7.5 MiB/s rd, 39 MiB/s wr, 590 op/s 2024-03-29T04:58:14.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:14 smithi181 ceph-mon[101970]: Reconfiguring mgr.y (monmap changed)... 2024-03-29T04:58:14.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:14 smithi181 ceph-mon[101970]: Reconfiguring daemon mgr.y on smithi181 2024-03-29T04:58:14.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:14 smithi181 ceph-mon[101970]: failed to encode map e350 with expected crc 2024-03-29T04:58:14.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:14 smithi181 ceph-mon[101970]: failed to encode map e350 with expected crc 2024-03-29T04:58:14.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:14 smithi181 ceph-mon[101970]: osdmap e350: 8 total, 8 up, 8 in 2024-03-29T04:58:14.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:14 smithi181 ceph-mon[101970]: failed to encode map e350 with expected crc 2024-03-29T04:58:15.230 INFO:tasks.workunit.client.0.smithi181.stdout:migration_execute 2024-03-29T04:58:15.230 INFO:tasks.workunit.client.0.smithi181.stdout:write: a 10342766~6611555 2024-03-29T04:58:15.301 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 59616133~5915906 2024-03-29T04:58:15.341 INFO:tasks.workunit.client.0.smithi181.stdout:write: a 84769455~6639817 2024-03-29T04:58:15.389 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 46706274~2501767 2024-03-29T04:58:15.412 INFO:tasks.workunit.client.0.smithi181.stdout:write: a 36327848~7748199 2024-03-29T04:58:15.481 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 118952710~6879247 2024-03-29T04:58:15.530 INFO:tasks.workunit.client.0.smithi181.stdout:write: a 91222743~2689218 2024-03-29T04:58:15.569 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 146043729~1468132 2024-03-29T04:58:15.573 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: user0 2024-03-29T04:58:15.795 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[106466]: failed to encode map e351 with expected crc 2024-03-29T04:58:15.796 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[106466]: failed to encode map e351 with expected crc 2024-03-29T04:58:15.796 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[106466]: osdmap e351: 8 total, 8 up, 8 in 2024-03-29T04:58:15.796 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[106466]: failed to encode map e351 with expected crc 2024-03-29T04:58:15.796 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[106466]: failed to encode map e351 with expected crc 2024-03-29T04:58:15.796 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[106466]: failed to encode map e351 with expected crc 2024-03-29T04:58:15.796 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:15.796 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:15.796 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:58:15.796 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:58:15.796 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:15.797 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[101970]: failed to encode map e351 with expected crc 2024-03-29T04:58:15.797 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[101970]: failed to encode map e351 with expected crc 2024-03-29T04:58:15.797 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[101970]: osdmap e351: 8 total, 8 up, 8 in 2024-03-29T04:58:15.797 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[101970]: failed to encode map e351 with expected crc 2024-03-29T04:58:15.797 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[101970]: failed to encode map e351 with expected crc 2024-03-29T04:58:15.797 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[101970]: failed to encode map e351 with expected crc 2024-03-29T04:58:15.797 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:15.797 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:15.797 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:58:15.797 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:58:15.797 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:15 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:15.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:15 smithi187 ceph-mon[93536]: failed to encode map e351 with expected crc 2024-03-29T04:58:15.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:15 smithi187 ceph-mon[93536]: failed to encode map e351 with expected crc 2024-03-29T04:58:15.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:15 smithi187 ceph-mon[93536]: osdmap e351: 8 total, 8 up, 8 in 2024-03-29T04:58:15.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:15 smithi187 ceph-mon[93536]: failed to encode map e351 with expected crc 2024-03-29T04:58:15.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:15 smithi187 ceph-mon[93536]: failed to encode map e351 with expected crc 2024-03-29T04:58:15.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:15 smithi187 ceph-mon[93536]: failed to encode map e351 with expected crc 2024-03-29T04:58:15.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:15 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:15.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:15 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:15.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:15 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:58:15.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:15 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:58:15.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:15 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:16.768 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:16 smithi181 ceph-mon[106466]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:58:16.768 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:16 smithi181 ceph-mon[106466]: Reconfiguring daemon mon.c on smithi181 2024-03-29T04:58:16.768 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:16 smithi181 ceph-mon[106466]: pgmap v178: 137 pgs: 137 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 5.0 MiB/s rd, 26 MiB/s wr, 391 op/s 2024-03-29T04:58:16.768 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:16 smithi181 ceph-mon[101970]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:58:16.769 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:16 smithi181 ceph-mon[101970]: Reconfiguring daemon mon.c on smithi181 2024-03-29T04:58:16.769 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:16 smithi181 ceph-mon[101970]: pgmap v178: 137 pgs: 137 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 5.0 MiB/s rd, 26 MiB/s wr, 391 op/s 2024-03-29T04:58:16.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:16 smithi187 ceph-mon[93536]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:58:16.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:16 smithi187 ceph-mon[93536]: Reconfiguring daemon mon.c on smithi181 2024-03-29T04:58:16.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:16 smithi187 ceph-mon[93536]: pgmap v178: 137 pgs: 137 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 5.0 MiB/s rd, 26 MiB/s wr, 391 op/s 2024-03-29T04:58:17.564 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 168493400 2024-03-29T04:58:17.575 INFO:tasks.workunit.client.0.smithi181.stdout:write: b 10737291~7946959 2024-03-29T04:58:17.588 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 16195047~7805855 2024-03-29T04:58:17.619 INFO:tasks.workunit.client.0.smithi181.stdout:write: b 27176382~2823814 2024-03-29T04:58:17.623 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 162236857~6256543 2024-03-29T04:58:17.623 INFO:tasks.workunit.client.0.smithi181.stdout:write: b 129931639~3711194 2024-03-29T04:58:17.630 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 22807752~5203653 2024-03-29T04:58:17.674 INFO:tasks.workunit.client.0.smithi181.stdout:write: b 14433896~3483340 2024-03-29T04:58:17.675 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 5071806~3409363 2024-03-29T04:58:17.676 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: user1 2024-03-29T04:58:17.680 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[101970]: failed to encode map e352 with expected crc 2024-03-29T04:58:17.680 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[101970]: failed to encode map e352 with expected crc 2024-03-29T04:58:17.680 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[101970]: osdmap e352: 8 total, 8 up, 8 in 2024-03-29T04:58:17.680 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[101970]: failed to encode map e352 with expected crc 2024-03-29T04:58:17.680 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[101970]: failed to encode map e352 with expected crc 2024-03-29T04:58:17.680 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:17.681 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:17.681 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[101970]: Reconfiguring osd.0 (monmap changed)... 2024-03-29T04:58:17.681 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:58:17.681 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:17.681 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[101970]: Reconfiguring daemon osd.0 on smithi181 2024-03-29T04:58:17.681 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[106466]: failed to encode map e352 with expected crc 2024-03-29T04:58:17.681 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[106466]: failed to encode map e352 with expected crc 2024-03-29T04:58:17.681 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[106466]: osdmap e352: 8 total, 8 up, 8 in 2024-03-29T04:58:17.681 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[106466]: failed to encode map e352 with expected crc 2024-03-29T04:58:17.681 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[106466]: failed to encode map e352 with expected crc 2024-03-29T04:58:17.682 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:17.682 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:17.682 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[106466]: Reconfiguring osd.0 (monmap changed)... 2024-03-29T04:58:17.682 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:58:17.682 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:17.682 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:17 smithi181 ceph-mon[106466]: Reconfiguring daemon osd.0 on smithi181 2024-03-29T04:58:17.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:17 smithi187 ceph-mon[93536]: failed to encode map e352 with expected crc 2024-03-29T04:58:17.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:17 smithi187 ceph-mon[93536]: failed to encode map e352 with expected crc 2024-03-29T04:58:17.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:17 smithi187 ceph-mon[93536]: osdmap e352: 8 total, 8 up, 8 in 2024-03-29T04:58:17.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:17 smithi187 ceph-mon[93536]: failed to encode map e352 with expected crc 2024-03-29T04:58:17.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:17 smithi187 ceph-mon[93536]: failed to encode map e352 with expected crc 2024-03-29T04:58:17.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:17 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:17.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:17 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:17.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:17 smithi187 ceph-mon[93536]: Reconfiguring osd.0 (monmap changed)... 2024-03-29T04:58:17.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:17 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:58:17.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:17 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:17.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:17 smithi187 ceph-mon[93536]: Reconfiguring daemon osd.0 on smithi181 2024-03-29T04:58:18.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[101970]: pgmap v180: 137 pgs: 137 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2024-03-29T04:58:18.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[101970]: failed to encode map e353 with expected crc 2024-03-29T04:58:18.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[101970]: failed to encode map e353 with expected crc 2024-03-29T04:58:18.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[101970]: osdmap e353: 8 total, 8 up, 8 in 2024-03-29T04:58:18.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[101970]: failed to encode map e353 with expected crc 2024-03-29T04:58:18.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[101970]: failed to encode map e353 with expected crc 2024-03-29T04:58:18.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[101970]: failed to encode map e353 with expected crc 2024-03-29T04:58:18.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:18.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:18.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:58:18.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:18.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[106466]: pgmap v180: 137 pgs: 137 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2024-03-29T04:58:18.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[106466]: failed to encode map e353 with expected crc 2024-03-29T04:58:18.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[106466]: failed to encode map e353 with expected crc 2024-03-29T04:58:18.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[106466]: osdmap e353: 8 total, 8 up, 8 in 2024-03-29T04:58:18.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[106466]: failed to encode map e353 with expected crc 2024-03-29T04:58:18.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[106466]: failed to encode map e353 with expected crc 2024-03-29T04:58:18.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[106466]: failed to encode map e353 with expected crc 2024-03-29T04:58:18.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:18.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:18.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:58:18.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:18.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:58:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:58:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:58:18.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:18 smithi187 ceph-mon[93536]: pgmap v180: 137 pgs: 137 active+clean; 2.1 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail 2024-03-29T04:58:18.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:18 smithi187 ceph-mon[93536]: failed to encode map e353 with expected crc 2024-03-29T04:58:18.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:18 smithi187 ceph-mon[93536]: failed to encode map e353 with expected crc 2024-03-29T04:58:18.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:18 smithi187 ceph-mon[93536]: osdmap e353: 8 total, 8 up, 8 in 2024-03-29T04:58:18.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:18 smithi187 ceph-mon[93536]: failed to encode map e353 with expected crc 2024-03-29T04:58:18.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:18 smithi187 ceph-mon[93536]: failed to encode map e353 with expected crc 2024-03-29T04:58:18.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:18 smithi187 ceph-mon[93536]: failed to encode map e353 with expected crc 2024-03-29T04:58:18.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:18 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:18.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:18 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:18.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:18 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:58:18.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:18 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:19.211 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:58:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:58:18] "GET /metrics HTTP/1.1" 200 38714 "" "Prometheus/2.43.0" 2024-03-29T04:58:19.588 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 247059731 2024-03-29T04:58:19.604 INFO:tasks.workunit.client.0.smithi181.stdout:write: c 232587376~1809814 2024-03-29T04:58:19.611 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 153781170~5213405 2024-03-29T04:58:19.612 INFO:tasks.workunit.client.0.smithi181.stdout:write: c 222616262~4587659 2024-03-29T04:58:19.614 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 114155502~1387004 2024-03-29T04:58:19.614 INFO:tasks.workunit.client.0.smithi181.stdout:write: c 240860628~6199103 2024-03-29T04:58:19.623 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 168294834~4662265 2024-03-29T04:58:19.623 INFO:tasks.workunit.client.0.smithi181.stdout:write: c 61576004~4618380 2024-03-29T04:58:19.625 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 19447974~7121186 2024-03-29T04:58:19.625 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: user2 2024-03-29T04:58:19.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:19 smithi187 ceph-mon[93536]: Reconfiguring osd.1 (monmap changed)... 2024-03-29T04:58:19.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:19 smithi187 ceph-mon[93536]: Reconfiguring daemon osd.1 on smithi181 2024-03-29T04:58:19.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:19 smithi187 ceph-mon[93536]: failed to encode map e354 with expected crc 2024-03-29T04:58:19.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:19 smithi187 ceph-mon[93536]: failed to encode map e354 with expected crc 2024-03-29T04:58:19.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:19 smithi187 ceph-mon[93536]: osdmap e354: 8 total, 8 up, 8 in 2024-03-29T04:58:19.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:19 smithi187 ceph-mon[93536]: failed to encode map e354 with expected crc 2024-03-29T04:58:19.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:19 smithi187 ceph-mon[93536]: failed to encode map e354 with expected crc 2024-03-29T04:58:19.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:19 smithi181 ceph-mon[101970]: Reconfiguring osd.1 (monmap changed)... 2024-03-29T04:58:19.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:19 smithi181 ceph-mon[101970]: Reconfiguring daemon osd.1 on smithi181 2024-03-29T04:58:19.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:19 smithi181 ceph-mon[101970]: failed to encode map e354 with expected crc 2024-03-29T04:58:19.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:19 smithi181 ceph-mon[101970]: failed to encode map e354 with expected crc 2024-03-29T04:58:19.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:19 smithi181 ceph-mon[101970]: osdmap e354: 8 total, 8 up, 8 in 2024-03-29T04:58:19.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:19 smithi181 ceph-mon[101970]: failed to encode map e354 with expected crc 2024-03-29T04:58:19.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:19 smithi181 ceph-mon[101970]: failed to encode map e354 with expected crc 2024-03-29T04:58:19.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:19 smithi181 ceph-mon[106466]: Reconfiguring osd.1 (monmap changed)... 2024-03-29T04:58:19.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:19 smithi181 ceph-mon[106466]: Reconfiguring daemon osd.1 on smithi181 2024-03-29T04:58:19.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:19 smithi181 ceph-mon[106466]: failed to encode map e354 with expected crc 2024-03-29T04:58:19.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:19 smithi181 ceph-mon[106466]: failed to encode map e354 with expected crc 2024-03-29T04:58:19.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:19 smithi181 ceph-mon[106466]: osdmap e354: 8 total, 8 up, 8 in 2024-03-29T04:58:19.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:19 smithi181 ceph-mon[106466]: failed to encode map e354 with expected crc 2024-03-29T04:58:19.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:19 smithi181 ceph-mon[106466]: failed to encode map e354 with expected crc 2024-03-29T04:58:20.859 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[106466]: pgmap v183: 137 pgs: 137 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 36 MiB/s rd, 33 MiB/s wr, 959 op/s 2024-03-29T04:58:20.859 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[106466]: failed to encode map e355 with expected crc 2024-03-29T04:58:20.859 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[106466]: failed to encode map e355 with expected crc 2024-03-29T04:58:20.859 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[106466]: osdmap e355: 8 total, 8 up, 8 in 2024-03-29T04:58:20.859 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[106466]: failed to encode map e355 with expected crc 2024-03-29T04:58:20.859 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[106466]: failed to encode map e355 with expected crc 2024-03-29T04:58:20.859 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[106466]: failed to encode map e355 with expected crc 2024-03-29T04:58:20.859 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:20.859 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:20.859 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:58:20.859 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:20.860 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[101970]: pgmap v183: 137 pgs: 137 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 36 MiB/s rd, 33 MiB/s wr, 959 op/s 2024-03-29T04:58:20.860 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[101970]: failed to encode map e355 with expected crc 2024-03-29T04:58:20.860 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[101970]: failed to encode map e355 with expected crc 2024-03-29T04:58:20.860 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[101970]: osdmap e355: 8 total, 8 up, 8 in 2024-03-29T04:58:20.860 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[101970]: failed to encode map e355 with expected crc 2024-03-29T04:58:20.860 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[101970]: failed to encode map e355 with expected crc 2024-03-29T04:58:20.860 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[101970]: failed to encode map e355 with expected crc 2024-03-29T04:58:20.860 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:20.860 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:20.860 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:58:20.860 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:20 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:20.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:20 smithi187 ceph-mon[93536]: pgmap v183: 137 pgs: 137 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 36 MiB/s rd, 33 MiB/s wr, 959 op/s 2024-03-29T04:58:20.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:20 smithi187 ceph-mon[93536]: failed to encode map e355 with expected crc 2024-03-29T04:58:20.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:20 smithi187 ceph-mon[93536]: failed to encode map e355 with expected crc 2024-03-29T04:58:20.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:20 smithi187 ceph-mon[93536]: osdmap e355: 8 total, 8 up, 8 in 2024-03-29T04:58:20.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:20 smithi187 ceph-mon[93536]: failed to encode map e355 with expected crc 2024-03-29T04:58:20.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:20 smithi187 ceph-mon[93536]: failed to encode map e355 with expected crc 2024-03-29T04:58:20.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:20 smithi187 ceph-mon[93536]: failed to encode map e355 with expected crc 2024-03-29T04:58:20.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:20 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:20.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:20 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:20.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:20 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:58:20.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:20 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:21.584 INFO:tasks.workunit.client.0.smithi181.stdout:write: d 136551425~1556392 2024-03-29T04:58:21.589 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 229451829~7276303 2024-03-29T04:58:21.590 INFO:tasks.workunit.client.0.smithi181.stdout:write: d 96707382~2811251 2024-03-29T04:58:21.593 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 103807977~6546247 2024-03-29T04:58:21.594 INFO:tasks.workunit.client.0.smithi181.stdout:write: d 148789084~7916034 2024-03-29T04:58:21.604 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 175333963~4128814 2024-03-29T04:58:21.606 INFO:tasks.workunit.client.0.smithi181.stdout:write: d 233898622~8159121 2024-03-29T04:58:21.625 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 47000103~2347879 2024-03-29T04:58:21.630 INFO:tasks.workunit.client.0.smithi181.stdout:snap_create: user3 2024-03-29T04:58:21.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:21 smithi181 ceph-mon[101970]: Reconfiguring osd.2 (monmap changed)... 2024-03-29T04:58:21.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:21 smithi181 ceph-mon[101970]: Reconfiguring daemon osd.2 on smithi181 2024-03-29T04:58:21.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:21 smithi181 ceph-mon[101970]: failed to encode map e356 with expected crc 2024-03-29T04:58:21.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:21 smithi181 ceph-mon[101970]: failed to encode map e356 with expected crc 2024-03-29T04:58:21.718 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:21 smithi181 ceph-mon[101970]: osdmap e356: 8 total, 8 up, 8 in 2024-03-29T04:58:21.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:21 smithi181 ceph-mon[106466]: Reconfiguring osd.2 (monmap changed)... 2024-03-29T04:58:21.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:21 smithi181 ceph-mon[106466]: Reconfiguring daemon osd.2 on smithi181 2024-03-29T04:58:21.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:21 smithi181 ceph-mon[106466]: failed to encode map e356 with expected crc 2024-03-29T04:58:21.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:21 smithi181 ceph-mon[106466]: failed to encode map e356 with expected crc 2024-03-29T04:58:21.718 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:21 smithi181 ceph-mon[106466]: osdmap e356: 8 total, 8 up, 8 in 2024-03-29T04:58:21.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:21 smithi187 ceph-mon[93536]: Reconfiguring osd.2 (monmap changed)... 2024-03-29T04:58:21.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:21 smithi187 ceph-mon[93536]: Reconfiguring daemon osd.2 on smithi181 2024-03-29T04:58:21.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:21 smithi187 ceph-mon[93536]: failed to encode map e356 with expected crc 2024-03-29T04:58:21.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:21 smithi187 ceph-mon[93536]: failed to encode map e356 with expected crc 2024-03-29T04:58:21.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:21 smithi187 ceph-mon[93536]: osdmap e356: 8 total, 8 up, 8 in 2024-03-29T04:58:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[101970]: pgmap v186: 137 pgs: 137 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 41 MiB/s rd, 38 MiB/s wr, 1.11k op/s 2024-03-29T04:58:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[101970]: failed to encode map e357 with expected crc 2024-03-29T04:58:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[101970]: failed to encode map e357 with expected crc 2024-03-29T04:58:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[101970]: osdmap e357: 8 total, 8 up, 8 in 2024-03-29T04:58:22.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[101970]: failed to encode map e357 with expected crc 2024-03-29T04:58:22.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[101970]: failed to encode map e357 with expected crc 2024-03-29T04:58:22.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:22.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:22.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[101970]: Reconfiguring osd.3 (monmap changed)... 2024-03-29T04:58:22.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:58:22.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:22.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[101970]: Reconfiguring daemon osd.3 on smithi181 2024-03-29T04:58:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[106466]: pgmap v186: 137 pgs: 137 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 41 MiB/s rd, 38 MiB/s wr, 1.11k op/s 2024-03-29T04:58:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[106466]: failed to encode map e357 with expected crc 2024-03-29T04:58:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[106466]: failed to encode map e357 with expected crc 2024-03-29T04:58:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[106466]: osdmap e357: 8 total, 8 up, 8 in 2024-03-29T04:58:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[106466]: failed to encode map e357 with expected crc 2024-03-29T04:58:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[106466]: failed to encode map e357 with expected crc 2024-03-29T04:58:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[106466]: Reconfiguring osd.3 (monmap changed)... 2024-03-29T04:58:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:58:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:22 smithi181 ceph-mon[106466]: Reconfiguring daemon osd.3 on smithi181 2024-03-29T04:58:22.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:22 smithi187 ceph-mon[93536]: pgmap v186: 137 pgs: 137 active+clean; 2.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 41 MiB/s rd, 38 MiB/s wr, 1.11k op/s 2024-03-29T04:58:22.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:22 smithi187 ceph-mon[93536]: failed to encode map e357 with expected crc 2024-03-29T04:58:22.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:22 smithi187 ceph-mon[93536]: failed to encode map e357 with expected crc 2024-03-29T04:58:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:22 smithi187 ceph-mon[93536]: osdmap e357: 8 total, 8 up, 8 in 2024-03-29T04:58:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:22 smithi187 ceph-mon[93536]: failed to encode map e357 with expected crc 2024-03-29T04:58:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:22 smithi187 ceph-mon[93536]: failed to encode map e357 with expected crc 2024-03-29T04:58:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:22 smithi187 ceph-mon[93536]: Reconfiguring osd.3 (monmap changed)... 2024-03-29T04:58:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:58:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:22 smithi187 ceph-mon[93536]: Reconfiguring daemon osd.3 on smithi181 2024-03-29T04:58:23.609 INFO:tasks.workunit.client.0.smithi181.stdout:write: 1 24705973~6176493 2024-03-29T04:58:23.614 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 26250096~3088246 2024-03-29T04:58:23.646 INFO:tasks.workunit.client.0.smithi181.stdout:write: 2 49411946~6176493 2024-03-29T04:58:23.649 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 50956069~3088246 2024-03-29T04:58:23.679 INFO:tasks.workunit.client.0.smithi181.stdout:write: 3 74117919~6176493 2024-03-29T04:58:23.681 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 75662042~3088246 2024-03-29T04:58:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:23 smithi181 ceph-mon[106466]: failed to encode map e358 with expected crc 2024-03-29T04:58:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:23 smithi181 ceph-mon[106466]: failed to encode map e358 with expected crc 2024-03-29T04:58:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:23 smithi181 ceph-mon[106466]: osdmap e358: 8 total, 8 up, 8 in 2024-03-29T04:58:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:23 smithi181 ceph-mon[106466]: failed to encode map e358 with expected crc 2024-03-29T04:58:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:23 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:23 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:23 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.pflrwm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:58:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:23 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:23.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:23 smithi181 ceph-mon[101970]: failed to encode map e358 with expected crc 2024-03-29T04:58:23.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:23 smithi181 ceph-mon[101970]: failed to encode map e358 with expected crc 2024-03-29T04:58:23.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:23 smithi181 ceph-mon[101970]: osdmap e358: 8 total, 8 up, 8 in 2024-03-29T04:58:23.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:23 smithi181 ceph-mon[101970]: failed to encode map e358 with expected crc 2024-03-29T04:58:23.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:23 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:23.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:23 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:23.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:23 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.pflrwm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:58:23.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:23 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:23.749 INFO:tasks.workunit.client.0.smithi181.stdout:write: 4 98823892~6176493 2024-03-29T04:58:23.752 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 100368015~3088246 2024-03-29T04:58:23.786 INFO:tasks.workunit.client.0.smithi181.stdout:write: 5 123529865~6176493 2024-03-29T04:58:23.788 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 125073988~3088246 2024-03-29T04:58:23.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:23 smithi187 ceph-mon[93536]: failed to encode map e358 with expected crc 2024-03-29T04:58:23.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:23 smithi187 ceph-mon[93536]: failed to encode map e358 with expected crc 2024-03-29T04:58:23.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:23 smithi187 ceph-mon[93536]: osdmap e358: 8 total, 8 up, 8 in 2024-03-29T04:58:23.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:23 smithi187 ceph-mon[93536]: failed to encode map e358 with expected crc 2024-03-29T04:58:23.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:23 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:23.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:23 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:23.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:23 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi181.pflrwm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:58:23.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:23 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:23.922 INFO:tasks.workunit.client.0.smithi181.stdout:migration_commit 2024-03-29T04:58:24.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:24 smithi187 ceph-mon[93536]: pgmap v189: 137 pgs: 137 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 5.1 MiB/s rd, 34 MiB/s wr, 298 op/s 2024-03-29T04:58:24.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:24 smithi187 ceph-mon[93536]: Reconfiguring mds.foo.smithi181.pflrwm (monmap changed)... 2024-03-29T04:58:24.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:24 smithi187 ceph-mon[93536]: Reconfiguring daemon mds.foo.smithi181.pflrwm on smithi181 2024-03-29T04:58:24.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:24 smithi187 ceph-mon[93536]: failed to encode map e359 with expected crc 2024-03-29T04:58:24.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:24 smithi187 ceph-mon[93536]: failed to encode map e359 with expected crc 2024-03-29T04:58:24.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:24 smithi187 ceph-mon[93536]: osdmap e359: 8 total, 8 up, 8 in 2024-03-29T04:58:24.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:24 smithi187 ceph-mon[93536]: failed to encode map e359 with expected crc 2024-03-29T04:58:24.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:24 smithi187 ceph-mon[93536]: failed to encode map e359 with expected crc 2024-03-29T04:58:24.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:24 smithi187 ceph-mon[93536]: failed to encode map e359 with expected crc 2024-03-29T04:58:24.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[101970]: pgmap v189: 137 pgs: 137 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 5.1 MiB/s rd, 34 MiB/s wr, 298 op/s 2024-03-29T04:58:24.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[101970]: Reconfiguring mds.foo.smithi181.pflrwm (monmap changed)... 2024-03-29T04:58:24.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[101970]: Reconfiguring daemon mds.foo.smithi181.pflrwm on smithi181 2024-03-29T04:58:24.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[101970]: failed to encode map e359 with expected crc 2024-03-29T04:58:24.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[101970]: failed to encode map e359 with expected crc 2024-03-29T04:58:24.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[101970]: osdmap e359: 8 total, 8 up, 8 in 2024-03-29T04:58:24.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[101970]: failed to encode map e359 with expected crc 2024-03-29T04:58:24.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[101970]: failed to encode map e359 with expected crc 2024-03-29T04:58:24.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[101970]: failed to encode map e359 with expected crc 2024-03-29T04:58:24.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[106466]: pgmap v189: 137 pgs: 137 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 5.1 MiB/s rd, 34 MiB/s wr, 298 op/s 2024-03-29T04:58:24.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[106466]: Reconfiguring mds.foo.smithi181.pflrwm (monmap changed)... 2024-03-29T04:58:24.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[106466]: Reconfiguring daemon mds.foo.smithi181.pflrwm on smithi181 2024-03-29T04:58:24.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[106466]: failed to encode map e359 with expected crc 2024-03-29T04:58:24.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[106466]: failed to encode map e359 with expected crc 2024-03-29T04:58:24.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[106466]: osdmap e359: 8 total, 8 up, 8 in 2024-03-29T04:58:24.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[106466]: failed to encode map e359 with expected crc 2024-03-29T04:58:24.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[106466]: failed to encode map e359 with expected crc 2024-03-29T04:58:24.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:24 smithi181 ceph-mon[106466]: failed to encode map e359 with expected crc 2024-03-29T04:58:25.497 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMigration.StressLive (18737 ms) 2024-03-29T04:58:26.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:26 smithi187 ceph-mon[93536]: pgmap v191: 137 pgs: 137 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 4.5 MiB/s rd, 30 MiB/s wr, 260 op/s 2024-03-29T04:58:26.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:26.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:26.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:26 smithi187 ceph-mon[93536]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:58:26.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:58:26.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:58:26.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:26.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:26 smithi187 ceph-mon[93536]: Reconfiguring daemon mon.b on smithi187 2024-03-29T04:58:26.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[106466]: pgmap v191: 137 pgs: 137 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 4.5 MiB/s rd, 30 MiB/s wr, 260 op/s 2024-03-29T04:58:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[106466]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:58:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:58:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:58:26.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:26.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[106466]: Reconfiguring daemon mon.b on smithi187 2024-03-29T04:58:26.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:26.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[101970]: pgmap v191: 137 pgs: 137 active+clean; 2.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 4.5 MiB/s rd, 30 MiB/s wr, 260 op/s 2024-03-29T04:58:26.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:26.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:26.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[101970]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:58:26.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:58:26.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:58:26.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:26.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[101970]: Reconfiguring daemon mon.b on smithi187 2024-03-29T04:58:26.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:27.462 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 57 tests from TestMigration (275553 ms total) 2024-03-29T04:58:27.463 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T04:58:27.463 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 6 tests from TestGroup 2024-03-29T04:58:27.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:27 smithi181 ceph-mon[106466]: failed to encode map e360 with expected crc 2024-03-29T04:58:27.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:27 smithi181 ceph-mon[106466]: osdmap e360: 8 total, 8 up, 8 in 2024-03-29T04:58:27.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:27 smithi181 ceph-mon[106466]: failed to encode map e360 with expected crc 2024-03-29T04:58:27.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:27 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:27.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:27 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:27.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:27 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:58:27.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:27 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:58:27.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:27 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:27 smithi181 ceph-mon[101970]: failed to encode map e360 with expected crc 2024-03-29T04:58:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:27 smithi181 ceph-mon[101970]: osdmap e360: 8 total, 8 up, 8 in 2024-03-29T04:58:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:27 smithi181 ceph-mon[101970]: failed to encode map e360 with expected crc 2024-03-29T04:58:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:27 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:27 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:27 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:58:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:27 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:58:27.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:27 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:27.762 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:27 smithi187 ceph-mon[93536]: failed to encode map e360 with expected crc 2024-03-29T04:58:27.762 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:27 smithi187 ceph-mon[93536]: osdmap e360: 8 total, 8 up, 8 in 2024-03-29T04:58:27.762 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:27 smithi187 ceph-mon[93536]: failed to encode map e360 with expected crc 2024-03-29T04:58:27.762 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:27 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:27.762 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:27 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:27.762 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:27 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:58:27.762 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:27 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:58:27.762 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:27 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:28.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-mon[101970]: Reconfiguring mgr.x (monmap changed)... 2024-03-29T04:58:28.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-mon[101970]: Reconfiguring daemon mgr.x on smithi187 2024-03-29T04:58:28.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-mon[101970]: pgmap v193: 105 pgs: 105 active+clean; 2.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 7.5 MiB/s wr, 1 op/s 2024-03-29T04:58:28.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-mon[101970]: failed to encode map e361 with expected crc 2024-03-29T04:58:28.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-mon[101970]: failed to encode map e361 with expected crc 2024-03-29T04:58:28.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-mon[101970]: osdmap e361: 8 total, 8 up, 8 in 2024-03-29T04:58:28.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-mon[101970]: failed to encode map e360 with expected crc 2024-03-29T04:58:28.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-mon[101970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:58:28.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-mon[106466]: Reconfiguring mgr.x (monmap changed)... 2024-03-29T04:58:28.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-mon[106466]: Reconfiguring daemon mgr.x on smithi187 2024-03-29T04:58:28.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-mon[106466]: pgmap v193: 105 pgs: 105 active+clean; 2.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 7.5 MiB/s wr, 1 op/s 2024-03-29T04:58:28.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-mon[106466]: failed to encode map e361 with expected crc 2024-03-29T04:58:28.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-mon[106466]: failed to encode map e361 with expected crc 2024-03-29T04:58:28.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-mon[106466]: osdmap e361: 8 total, 8 up, 8 in 2024-03-29T04:58:28.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-mon[106466]: failed to encode map e360 with expected crc 2024-03-29T04:58:28.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-mon[106466]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:58:28.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:58:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:58:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:58:28.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:28 smithi187 ceph-mon[93536]: Reconfiguring mgr.x (monmap changed)... 2024-03-29T04:58:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:28 smithi187 ceph-mon[93536]: Reconfiguring daemon mgr.x on smithi187 2024-03-29T04:58:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:28 smithi187 ceph-mon[93536]: pgmap v193: 105 pgs: 105 active+clean; 2.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 7.5 MiB/s wr, 1 op/s 2024-03-29T04:58:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:28 smithi187 ceph-mon[93536]: failed to encode map e361 with expected crc 2024-03-29T04:58:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:28 smithi187 ceph-mon[93536]: failed to encode map e361 with expected crc 2024-03-29T04:58:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:28 smithi187 ceph-mon[93536]: osdmap e361: 8 total, 8 up, 8 in 2024-03-29T04:58:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:28 smithi187 ceph-mon[93536]: failed to encode map e360 with expected crc 2024-03-29T04:58:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:28 smithi187 ceph-mon[93536]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:58:29.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:58:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:58:28] "GET /metrics HTTP/1.1" 200 38728 "" "Prometheus/2.43.0" 2024-03-29T04:58:29.464 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestGroup.group_create 2024-03-29T04:58:29.588 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:29 smithi187 ceph-mon[93536]: failed to encode map e360 with expected crc 2024-03-29T04:58:29.588 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:29 smithi187 ceph-mon[93536]: failed to encode map e361 with expected crc 2024-03-29T04:58:29.588 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:29 smithi187 ceph-mon[93536]: failed to encode map e362 with expected crc 2024-03-29T04:58:29.588 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:29 smithi187 ceph-mon[93536]: failed to encode map e362 with expected crc 2024-03-29T04:58:29.588 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:29 smithi187 ceph-mon[93536]: osdmap e362: 8 total, 8 up, 8 in 2024-03-29T04:58:29.588 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:29 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1071294445' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:58:29.588 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:29 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:29.588 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:29 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:29.589 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:29 smithi187 ceph-mon[93536]: Reconfiguring osd.4 (monmap changed)... 2024-03-29T04:58:29.589 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:29 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:58:29.589 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:29 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:29.589 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:29 smithi187 ceph-mon[93536]: Reconfiguring daemon osd.4 on smithi187 2024-03-29T04:58:29.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[101970]: failed to encode map e360 with expected crc 2024-03-29T04:58:29.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[101970]: failed to encode map e361 with expected crc 2024-03-29T04:58:29.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[101970]: failed to encode map e362 with expected crc 2024-03-29T04:58:29.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[101970]: failed to encode map e362 with expected crc 2024-03-29T04:58:29.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[101970]: osdmap e362: 8 total, 8 up, 8 in 2024-03-29T04:58:29.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1071294445' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:58:29.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:29.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:29.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[101970]: Reconfiguring osd.4 (monmap changed)... 2024-03-29T04:58:29.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:58:29.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:29.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[101970]: Reconfiguring daemon osd.4 on smithi187 2024-03-29T04:58:29.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[106466]: failed to encode map e360 with expected crc 2024-03-29T04:58:29.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[106466]: failed to encode map e361 with expected crc 2024-03-29T04:58:29.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[106466]: failed to encode map e362 with expected crc 2024-03-29T04:58:29.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[106466]: failed to encode map e362 with expected crc 2024-03-29T04:58:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[106466]: osdmap e362: 8 total, 8 up, 8 in 2024-03-29T04:58:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1071294445' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:58:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[106466]: Reconfiguring osd.4 (monmap changed)... 2024-03-29T04:58:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:58:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:29 smithi181 ceph-mon[106466]: Reconfiguring daemon osd.4 on smithi187 2024-03-29T04:58:30.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: pgmap v196: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2024-03-29T04:58:30.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1071294445' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:58:30.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: osdmap e363: 8 total, 8 up, 8 in 2024-03-29T04:58:30.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:58:30.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:30.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:30.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:58:30.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:30 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: pgmap v196: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2024-03-29T04:58:30.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1071294445' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:58:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: osdmap e363: 8 total, 8 up, 8 in 2024-03-29T04:58:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:58:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:58:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: pgmap v196: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2024-03-29T04:58:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1071294445' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:58:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: osdmap e363: 8 total, 8 up, 8 in 2024-03-29T04:58:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: failed to encode map e363 with expected crc 2024-03-29T04:58:30.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:58:30.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:30.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:30.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:58:30.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:30 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:31.515 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestGroup.group_create (2050 ms) 2024-03-29T04:58:31.515 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestGroup.group_createPP 2024-03-29T04:58:31.534 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestGroup.group_createPP (20 ms) 2024-03-29T04:58:31.534 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestGroup.add_image 2024-03-29T04:58:31.569 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestGroup.add_image (35 ms) 2024-03-29T04:58:31.569 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestGroup.add_imagePP 2024-03-29T04:58:31.605 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestGroup.add_imagePP (35 ms) 2024-03-29T04:58:31.605 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestGroup.add_snapshot 2024-03-29T04:58:31.787 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:31 smithi187 ceph-mon[93536]: Reconfiguring osd.5 (monmap changed)... 2024-03-29T04:58:31.787 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:31 smithi187 ceph-mon[93536]: Reconfiguring daemon osd.5 on smithi187 2024-03-29T04:58:31.787 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:31 smithi187 ceph-mon[93536]: failed to encode map e364 with expected crc 2024-03-29T04:58:31.787 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:31 smithi187 ceph-mon[93536]: failed to encode map e364 with expected crc 2024-03-29T04:58:31.788 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:31 smithi187 ceph-mon[93536]: osdmap e364: 8 total, 8 up, 8 in 2024-03-29T04:58:31.788 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:31 smithi187 ceph-mon[93536]: failed to encode map e364 with expected crc 2024-03-29T04:58:31.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:31 smithi181 ceph-mon[101970]: Reconfiguring osd.5 (monmap changed)... 2024-03-29T04:58:31.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:31 smithi181 ceph-mon[101970]: Reconfiguring daemon osd.5 on smithi187 2024-03-29T04:58:31.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:31 smithi181 ceph-mon[101970]: failed to encode map e364 with expected crc 2024-03-29T04:58:31.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:31 smithi181 ceph-mon[101970]: failed to encode map e364 with expected crc 2024-03-29T04:58:31.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:31 smithi181 ceph-mon[101970]: osdmap e364: 8 total, 8 up, 8 in 2024-03-29T04:58:31.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:31 smithi181 ceph-mon[101970]: failed to encode map e364 with expected crc 2024-03-29T04:58:31.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:31 smithi181 ceph-mon[106466]: Reconfiguring osd.5 (monmap changed)... 2024-03-29T04:58:31.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:31 smithi181 ceph-mon[106466]: Reconfiguring daemon osd.5 on smithi187 2024-03-29T04:58:31.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:31 smithi181 ceph-mon[106466]: failed to encode map e364 with expected crc 2024-03-29T04:58:31.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:31 smithi181 ceph-mon[106466]: failed to encode map e364 with expected crc 2024-03-29T04:58:31.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:31 smithi181 ceph-mon[106466]: osdmap e364: 8 total, 8 up, 8 in 2024-03-29T04:58:31.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:31 smithi181 ceph-mon[106466]: failed to encode map e364 with expected crc 2024-03-29T04:58:32.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:32 smithi187 ceph-mon[93536]: pgmap v199: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2024-03-29T04:58:32.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:32 smithi187 ceph-mon[93536]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:58:32.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:32 smithi187 ceph-mon[93536]: failed to encode map e365 with expected crc 2024-03-29T04:58:32.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:32 smithi187 ceph-mon[93536]: failed to encode map e365 with expected crc 2024-03-29T04:58:32.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:32 smithi187 ceph-mon[93536]: osdmap e365: 8 total, 8 up, 8 in 2024-03-29T04:58:32.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:32 smithi187 ceph-mon[93536]: failed to encode map e365 with expected crc 2024-03-29T04:58:32.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:32 smithi187 ceph-mon[93536]: failed to encode map e365 with expected crc 2024-03-29T04:58:32.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:32 smithi187 ceph-mon[93536]: failed to encode map e365 with expected crc 2024-03-29T04:58:32.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:32 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:32.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:32 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:32.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[101970]: pgmap v199: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2024-03-29T04:58:32.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[101970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:58:32.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[101970]: failed to encode map e365 with expected crc 2024-03-29T04:58:32.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[101970]: failed to encode map e365 with expected crc 2024-03-29T04:58:32.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[101970]: osdmap e365: 8 total, 8 up, 8 in 2024-03-29T04:58:32.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[101970]: failed to encode map e365 with expected crc 2024-03-29T04:58:32.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[101970]: failed to encode map e365 with expected crc 2024-03-29T04:58:32.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[101970]: failed to encode map e365 with expected crc 2024-03-29T04:58:32.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:32.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:32.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[106466]: pgmap v199: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2024-03-29T04:58:32.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[106466]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:58:32.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[106466]: failed to encode map e365 with expected crc 2024-03-29T04:58:32.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[106466]: failed to encode map e365 with expected crc 2024-03-29T04:58:32.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[106466]: osdmap e365: 8 total, 8 up, 8 in 2024-03-29T04:58:32.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[106466]: failed to encode map e365 with expected crc 2024-03-29T04:58:32.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[106466]: failed to encode map e365 with expected crc 2024-03-29T04:58:32.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[106466]: failed to encode map e365 with expected crc 2024-03-29T04:58:32.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:32.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:32 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:33.856 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:33 smithi187 ceph-mon[93536]: failed to encode map e366 with expected crc 2024-03-29T04:58:33.856 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:33 smithi187 ceph-mon[93536]: failed to encode map e366 with expected crc 2024-03-29T04:58:33.856 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:33 smithi187 ceph-mon[93536]: osdmap e366: 8 total, 8 up, 8 in 2024-03-29T04:58:33.857 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:33 smithi187 ceph-mon[93536]: failed to encode map e366 with expected crc 2024-03-29T04:58:33.857 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:33 smithi187 ceph-mon[93536]: failed to encode map e366 with expected crc 2024-03-29T04:58:33.857 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:33 smithi187 ceph-mon[93536]: Reconfiguring osd.6 (monmap changed)... 2024-03-29T04:58:33.857 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:33 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:58:33.857 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:33 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:33.857 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:33 smithi187 ceph-mon[93536]: Reconfiguring daemon osd.6 on smithi187 2024-03-29T04:58:33.857 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:33 smithi187 ceph-mon[93536]: failed to encode map e366 with expected crc 2024-03-29T04:58:33.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[101970]: failed to encode map e366 with expected crc 2024-03-29T04:58:33.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[101970]: failed to encode map e366 with expected crc 2024-03-29T04:58:33.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[101970]: osdmap e366: 8 total, 8 up, 8 in 2024-03-29T04:58:33.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[101970]: failed to encode map e366 with expected crc 2024-03-29T04:58:33.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[101970]: failed to encode map e366 with expected crc 2024-03-29T04:58:33.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[101970]: Reconfiguring osd.6 (monmap changed)... 2024-03-29T04:58:33.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:58:33.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:33.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[101970]: Reconfiguring daemon osd.6 on smithi187 2024-03-29T04:58:33.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[101970]: failed to encode map e366 with expected crc 2024-03-29T04:58:33.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[106466]: failed to encode map e366 with expected crc 2024-03-29T04:58:33.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[106466]: failed to encode map e366 with expected crc 2024-03-29T04:58:33.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[106466]: osdmap e366: 8 total, 8 up, 8 in 2024-03-29T04:58:33.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[106466]: failed to encode map e366 with expected crc 2024-03-29T04:58:33.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[106466]: failed to encode map e366 with expected crc 2024-03-29T04:58:33.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[106466]: Reconfiguring osd.6 (monmap changed)... 2024-03-29T04:58:33.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:58:33.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:33.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[106466]: Reconfiguring daemon osd.6 on smithi187 2024-03-29T04:58:33.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:33 smithi181 ceph-mon[106466]: failed to encode map e366 with expected crc 2024-03-29T04:58:34.791 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:34 smithi187 ceph-mon[93536]: pgmap v202: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s rd, 13 KiB/s wr, 107 op/s 2024-03-29T04:58:34.791 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:34 smithi187 ceph-mon[93536]: failed to encode map e367 with expected crc 2024-03-29T04:58:34.791 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:34 smithi187 ceph-mon[93536]: failed to encode map e367 with expected crc 2024-03-29T04:58:34.791 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:34 smithi187 ceph-mon[93536]: osdmap e367: 8 total, 8 up, 8 in 2024-03-29T04:58:34.791 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:34 smithi187 ceph-mon[93536]: failed to encode map e367 with expected crc 2024-03-29T04:58:34.792 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:34 smithi187 ceph-mon[93536]: failed to encode map e367 with expected crc 2024-03-29T04:58:34.792 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:34 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:34.792 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:34 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:34.792 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:34 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:58:34.792 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:34 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:34.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[101970]: pgmap v202: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s rd, 13 KiB/s wr, 107 op/s 2024-03-29T04:58:34.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[101970]: failed to encode map e367 with expected crc 2024-03-29T04:58:34.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[101970]: failed to encode map e367 with expected crc 2024-03-29T04:58:34.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[101970]: osdmap e367: 8 total, 8 up, 8 in 2024-03-29T04:58:34.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[101970]: failed to encode map e367 with expected crc 2024-03-29T04:58:34.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[101970]: failed to encode map e367 with expected crc 2024-03-29T04:58:34.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:34.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:34.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:58:34.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:34.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[106466]: pgmap v202: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s rd, 13 KiB/s wr, 107 op/s 2024-03-29T04:58:34.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[106466]: failed to encode map e367 with expected crc 2024-03-29T04:58:34.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[106466]: failed to encode map e367 with expected crc 2024-03-29T04:58:34.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[106466]: osdmap e367: 8 total, 8 up, 8 in 2024-03-29T04:58:34.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[106466]: failed to encode map e367 with expected crc 2024-03-29T04:58:34.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[106466]: failed to encode map e367 with expected crc 2024-03-29T04:58:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:58:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:34 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:35.663 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:35 smithi187 ceph-mon[93536]: Reconfiguring osd.7 (monmap changed)... 2024-03-29T04:58:35.664 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:35 smithi187 ceph-mon[93536]: Reconfiguring daemon osd.7 on smithi187 2024-03-29T04:58:35.664 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:35 smithi187 ceph-mon[93536]: failed to encode map e368 with expected crc 2024-03-29T04:58:35.664 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:35 smithi187 ceph-mon[93536]: failed to encode map e368 with expected crc 2024-03-29T04:58:35.664 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:35 smithi187 ceph-mon[93536]: osdmap e368: 8 total, 8 up, 8 in 2024-03-29T04:58:35.664 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:35 smithi187 ceph-mon[93536]: failed to encode map e368 with expected crc 2024-03-29T04:58:35.664 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:35 smithi187 ceph-mon[93536]: failed to encode map e368 with expected crc 2024-03-29T04:58:35.664 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:35 smithi187 ceph-mon[93536]: failed to encode map e368 with expected crc 2024-03-29T04:58:35.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:35 smithi181 ceph-mon[101970]: Reconfiguring osd.7 (monmap changed)... 2024-03-29T04:58:35.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:35 smithi181 ceph-mon[101970]: Reconfiguring daemon osd.7 on smithi187 2024-03-29T04:58:35.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:35 smithi181 ceph-mon[101970]: failed to encode map e368 with expected crc 2024-03-29T04:58:35.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:35 smithi181 ceph-mon[101970]: failed to encode map e368 with expected crc 2024-03-29T04:58:35.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:35 smithi181 ceph-mon[101970]: osdmap e368: 8 total, 8 up, 8 in 2024-03-29T04:58:35.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:35 smithi181 ceph-mon[101970]: failed to encode map e368 with expected crc 2024-03-29T04:58:35.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:35 smithi181 ceph-mon[101970]: failed to encode map e368 with expected crc 2024-03-29T04:58:35.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:35 smithi181 ceph-mon[101970]: failed to encode map e368 with expected crc 2024-03-29T04:58:35.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:35 smithi181 ceph-mon[106466]: Reconfiguring osd.7 (monmap changed)... 2024-03-29T04:58:35.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:35 smithi181 ceph-mon[106466]: Reconfiguring daemon osd.7 on smithi187 2024-03-29T04:58:35.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:35 smithi181 ceph-mon[106466]: failed to encode map e368 with expected crc 2024-03-29T04:58:35.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:35 smithi181 ceph-mon[106466]: failed to encode map e368 with expected crc 2024-03-29T04:58:35.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:35 smithi181 ceph-mon[106466]: osdmap e368: 8 total, 8 up, 8 in 2024-03-29T04:58:35.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:35 smithi181 ceph-mon[106466]: failed to encode map e368 with expected crc 2024-03-29T04:58:35.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:35 smithi181 ceph-mon[106466]: failed to encode map e368 with expected crc 2024-03-29T04:58:35.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:35 smithi181 ceph-mon[106466]: failed to encode map e368 with expected crc 2024-03-29T04:58:36.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:36 smithi187 ceph-mon[93536]: pgmap v205: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s rd, 13 KiB/s wr, 107 op/s 2024-03-29T04:58:36.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:36 smithi187 ceph-mon[93536]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:36 smithi187 ceph-mon[93536]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:36 smithi187 ceph-mon[93536]: osdmap e369: 8 total, 8 up, 8 in 2024-03-29T04:58:36.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:36 smithi187 ceph-mon[93536]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:36 smithi187 ceph-mon[93536]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:36 smithi187 ceph-mon[93536]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:36 smithi187 ceph-mon[93536]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:36.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:36.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:36 smithi187 ceph-mon[93536]: Reconfiguring mds.foo.smithi187.vcsnqx (monmap changed)... 2024-03-29T04:58:36.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi187.vcsnqx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:58:36.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:36.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:36 smithi187 ceph-mon[93536]: Reconfiguring daemon mds.foo.smithi187.vcsnqx on smithi187 2024-03-29T04:58:36.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[101970]: pgmap v205: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s rd, 13 KiB/s wr, 107 op/s 2024-03-29T04:58:36.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[101970]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[101970]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[101970]: osdmap e369: 8 total, 8 up, 8 in 2024-03-29T04:58:36.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[101970]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[101970]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[101970]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[101970]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:36.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:36.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[101970]: Reconfiguring mds.foo.smithi187.vcsnqx (monmap changed)... 2024-03-29T04:58:36.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi187.vcsnqx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:58:36.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:36.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[101970]: Reconfiguring daemon mds.foo.smithi187.vcsnqx on smithi187 2024-03-29T04:58:36.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[106466]: pgmap v205: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s rd, 13 KiB/s wr, 107 op/s 2024-03-29T04:58:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[106466]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[106466]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[106466]: osdmap e369: 8 total, 8 up, 8 in 2024-03-29T04:58:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[106466]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[106466]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[106466]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[106466]: failed to encode map e369 with expected crc 2024-03-29T04:58:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[106466]: Reconfiguring mds.foo.smithi187.vcsnqx (monmap changed)... 2024-03-29T04:58:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi187.vcsnqx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:58:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:36 smithi181 ceph-mon[106466]: Reconfiguring daemon mds.foo.smithi187.vcsnqx on smithi187 2024-03-29T04:58:37.498 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestGroup.add_snapshot (5893 ms) 2024-03-29T04:58:37.498 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestGroup.add_snapshotPP 2024-03-29T04:58:37.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:37 smithi187 ceph-mon[93536]: failed to encode map e370 with expected crc 2024-03-29T04:58:37.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:37 smithi187 ceph-mon[93536]: failed to encode map e370 with expected crc 2024-03-29T04:58:37.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:37 smithi187 ceph-mon[93536]: osdmap e370: 8 total, 8 up, 8 in 2024-03-29T04:58:37.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:37 smithi187 ceph-mon[93536]: failed to encode map e370 with expected crc 2024-03-29T04:58:37.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:37 smithi187 ceph-mon[93536]: failed to encode map e370 with expected crc 2024-03-29T04:58:37.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:37 smithi187 ceph-mon[93536]: osdmap e371: 8 total, 8 up, 8 in 2024-03-29T04:58:37.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:37 smithi181 ceph-mon[101970]: failed to encode map e370 with expected crc 2024-03-29T04:58:37.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:37 smithi181 ceph-mon[101970]: failed to encode map e370 with expected crc 2024-03-29T04:58:37.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:37 smithi181 ceph-mon[101970]: osdmap e370: 8 total, 8 up, 8 in 2024-03-29T04:58:37.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:37 smithi181 ceph-mon[101970]: failed to encode map e370 with expected crc 2024-03-29T04:58:37.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:37 smithi181 ceph-mon[101970]: failed to encode map e370 with expected crc 2024-03-29T04:58:37.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:37 smithi181 ceph-mon[101970]: osdmap e371: 8 total, 8 up, 8 in 2024-03-29T04:58:37.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:37 smithi181 ceph-mon[106466]: failed to encode map e370 with expected crc 2024-03-29T04:58:37.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:37 smithi181 ceph-mon[106466]: failed to encode map e370 with expected crc 2024-03-29T04:58:37.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:37 smithi181 ceph-mon[106466]: osdmap e370: 8 total, 8 up, 8 in 2024-03-29T04:58:37.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:37 smithi181 ceph-mon[106466]: failed to encode map e370 with expected crc 2024-03-29T04:58:37.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:37 smithi181 ceph-mon[106466]: failed to encode map e370 with expected crc 2024-03-29T04:58:37.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:37 smithi181 ceph-mon[106466]: osdmap e371: 8 total, 8 up, 8 in 2024-03-29T04:58:38.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[106466]: pgmap v208: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:38.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[106466]: failed to encode map e371 with expected crc 2024-03-29T04:58:38.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[106466]: failed to encode map e371 with expected crc 2024-03-29T04:58:38.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[106466]: failed to encode map e371 with expected crc 2024-03-29T04:58:38.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[106466]: failed to encode map e371 with expected crc 2024-03-29T04:58:38.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[106466]: failed to encode map e371 with expected crc 2024-03-29T04:58:38.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:38.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:38.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:58:38.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:38.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:58:38.855 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[106466]: pgmap v210: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 9.9 KiB/s wr, 165 op/s 2024-03-29T04:58:38.856 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:38.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[101970]: pgmap v208: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:38.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[101970]: failed to encode map e371 with expected crc 2024-03-29T04:58:38.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[101970]: failed to encode map e371 with expected crc 2024-03-29T04:58:38.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[101970]: failed to encode map e371 with expected crc 2024-03-29T04:58:38.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[101970]: failed to encode map e371 with expected crc 2024-03-29T04:58:38.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[101970]: failed to encode map e371 with expected crc 2024-03-29T04:58:38.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:38.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:38.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:58:38.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:38.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:58:38.856 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[101970]: pgmap v210: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 9.9 KiB/s wr, 165 op/s 2024-03-29T04:58:38.857 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:38.883 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:58:38 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:58:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:58:38.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:38 smithi187 ceph-mon[93536]: pgmap v208: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:38.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:38 smithi187 ceph-mon[93536]: failed to encode map e371 with expected crc 2024-03-29T04:58:38.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:38 smithi187 ceph-mon[93536]: failed to encode map e371 with expected crc 2024-03-29T04:58:38.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:38 smithi187 ceph-mon[93536]: failed to encode map e371 with expected crc 2024-03-29T04:58:38.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:38 smithi187 ceph-mon[93536]: failed to encode map e371 with expected crc 2024-03-29T04:58:38.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:38 smithi187 ceph-mon[93536]: failed to encode map e371 with expected crc 2024-03-29T04:58:38.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:38.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:38.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:58:38.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:58:38.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:58:38.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:38 smithi187 ceph-mon[93536]: pgmap v210: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 9.9 KiB/s wr, 165 op/s 2024-03-29T04:58:38.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:39.120 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:58:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:58:38] "GET /metrics HTTP/1.1" 200 38728 "" "Prometheus/2.43.0" 2024-03-29T04:58:39.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:39 smithi187 ceph-mon[93536]: failed to encode map e372 with expected crc 2024-03-29T04:58:39.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:39 smithi187 ceph-mon[93536]: failed to encode map e372 with expected crc 2024-03-29T04:58:39.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:39 smithi187 ceph-mon[93536]: osdmap e372: 8 total, 8 up, 8 in 2024-03-29T04:58:39.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:39 smithi187 ceph-mon[93536]: failed to encode map e372 with expected crc 2024-03-29T04:58:39.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:39 smithi187 ceph-mon[93536]: failed to encode map e372 with expected crc 2024-03-29T04:58:39.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:39 smithi181 ceph-mon[106466]: failed to encode map e372 with expected crc 2024-03-29T04:58:39.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:39 smithi181 ceph-mon[106466]: failed to encode map e372 with expected crc 2024-03-29T04:58:39.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:39 smithi181 ceph-mon[106466]: osdmap e372: 8 total, 8 up, 8 in 2024-03-29T04:58:39.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:39 smithi181 ceph-mon[106466]: failed to encode map e372 with expected crc 2024-03-29T04:58:39.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:39 smithi181 ceph-mon[106466]: failed to encode map e372 with expected crc 2024-03-29T04:58:39.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:39 smithi181 ceph-mon[101970]: failed to encode map e372 with expected crc 2024-03-29T04:58:39.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:39 smithi181 ceph-mon[101970]: failed to encode map e372 with expected crc 2024-03-29T04:58:39.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:39 smithi181 ceph-mon[101970]: osdmap e372: 8 total, 8 up, 8 in 2024-03-29T04:58:39.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:39 smithi181 ceph-mon[101970]: failed to encode map e372 with expected crc 2024-03-29T04:58:39.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:39 smithi181 ceph-mon[101970]: failed to encode map e372 with expected crc 2024-03-29T04:58:40.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:40 smithi187 ceph-mon[93536]: failed to encode map e373 with expected crc 2024-03-29T04:58:40.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:40 smithi187 ceph-mon[93536]: failed to encode map e373 with expected crc 2024-03-29T04:58:40.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:40 smithi187 ceph-mon[93536]: osdmap e373: 8 total, 8 up, 8 in 2024-03-29T04:58:40.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:40 smithi187 ceph-mon[93536]: failed to encode map e373 with expected crc 2024-03-29T04:58:40.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:40 smithi187 ceph-mon[93536]: failed to encode map e373 with expected crc 2024-03-29T04:58:40.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:40 smithi187 ceph-mon[93536]: pgmap v213: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 129 KiB/s rd, 10 KiB/s wr, 166 op/s 2024-03-29T04:58:40.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:40 smithi181 ceph-mon[106466]: failed to encode map e373 with expected crc 2024-03-29T04:58:40.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:40 smithi181 ceph-mon[106466]: failed to encode map e373 with expected crc 2024-03-29T04:58:40.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:40 smithi181 ceph-mon[106466]: osdmap e373: 8 total, 8 up, 8 in 2024-03-29T04:58:40.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:40 smithi181 ceph-mon[106466]: failed to encode map e373 with expected crc 2024-03-29T04:58:40.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:40 smithi181 ceph-mon[106466]: failed to encode map e373 with expected crc 2024-03-29T04:58:40.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:40 smithi181 ceph-mon[106466]: pgmap v213: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 129 KiB/s rd, 10 KiB/s wr, 166 op/s 2024-03-29T04:58:40.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:40 smithi181 ceph-mon[101970]: failed to encode map e373 with expected crc 2024-03-29T04:58:40.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:40 smithi181 ceph-mon[101970]: failed to encode map e373 with expected crc 2024-03-29T04:58:40.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:40 smithi181 ceph-mon[101970]: osdmap e373: 8 total, 8 up, 8 in 2024-03-29T04:58:40.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:40 smithi181 ceph-mon[101970]: failed to encode map e373 with expected crc 2024-03-29T04:58:40.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:40 smithi181 ceph-mon[101970]: failed to encode map e373 with expected crc 2024-03-29T04:58:40.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:40 smithi181 ceph-mon[101970]: pgmap v213: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 129 KiB/s rd, 10 KiB/s wr, 166 op/s 2024-03-29T04:58:41.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:41 smithi187 ceph-mon[93536]: failed to encode map e373 with expected crc 2024-03-29T04:58:41.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:41 smithi187 ceph-mon[93536]: failed to encode map e374 with expected crc 2024-03-29T04:58:41.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:41 smithi187 ceph-mon[93536]: failed to encode map e374 with expected crc 2024-03-29T04:58:41.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:41 smithi187 ceph-mon[93536]: osdmap e374: 8 total, 8 up, 8 in 2024-03-29T04:58:41.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:41 smithi187 ceph-mon[93536]: failed to encode map e374 with expected crc 2024-03-29T04:58:41.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:41 smithi187 ceph-mon[93536]: failed to encode map e374 with expected crc 2024-03-29T04:58:41.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:41.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:41.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:41.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[101970]: failed to encode map e373 with expected crc 2024-03-29T04:58:41.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[101970]: failed to encode map e374 with expected crc 2024-03-29T04:58:41.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[101970]: failed to encode map e374 with expected crc 2024-03-29T04:58:41.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[101970]: osdmap e374: 8 total, 8 up, 8 in 2024-03-29T04:58:41.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[101970]: failed to encode map e374 with expected crc 2024-03-29T04:58:41.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[101970]: failed to encode map e374 with expected crc 2024-03-29T04:58:41.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:41.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:41.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[106466]: failed to encode map e373 with expected crc 2024-03-29T04:58:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[106466]: failed to encode map e374 with expected crc 2024-03-29T04:58:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[106466]: failed to encode map e374 with expected crc 2024-03-29T04:58:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[106466]: osdmap e374: 8 total, 8 up, 8 in 2024-03-29T04:58:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[106466]: failed to encode map e374 with expected crc 2024-03-29T04:58:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[106466]: failed to encode map e374 with expected crc 2024-03-29T04:58:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:42.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:42 smithi187 ceph-mon[93536]: failed to encode map e375 with expected crc 2024-03-29T04:58:42.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:42 smithi187 ceph-mon[93536]: failed to encode map e375 with expected crc 2024-03-29T04:58:42.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:42 smithi187 ceph-mon[93536]: osdmap e375: 8 total, 8 up, 8 in 2024-03-29T04:58:42.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:42 smithi187 ceph-mon[93536]: failed to encode map e375 with expected crc 2024-03-29T04:58:42.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:42 smithi187 ceph-mon[93536]: failed to encode map e375 with expected crc 2024-03-29T04:58:42.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:42 smithi187 ceph-mon[93536]: failed to encode map e375 with expected crc 2024-03-29T04:58:42.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:42 smithi187 ceph-mon[93536]: pgmap v216: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:42.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:42 smithi181 ceph-mon[106466]: failed to encode map e375 with expected crc 2024-03-29T04:58:42.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:42 smithi181 ceph-mon[106466]: failed to encode map e375 with expected crc 2024-03-29T04:58:42.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:42 smithi181 ceph-mon[106466]: osdmap e375: 8 total, 8 up, 8 in 2024-03-29T04:58:42.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:42 smithi181 ceph-mon[106466]: failed to encode map e375 with expected crc 2024-03-29T04:58:42.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:42 smithi181 ceph-mon[106466]: failed to encode map e375 with expected crc 2024-03-29T04:58:42.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:42 smithi181 ceph-mon[106466]: failed to encode map e375 with expected crc 2024-03-29T04:58:42.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:42 smithi181 ceph-mon[106466]: pgmap v216: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:42 smithi181 ceph-mon[101970]: failed to encode map e375 with expected crc 2024-03-29T04:58:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:42 smithi181 ceph-mon[101970]: failed to encode map e375 with expected crc 2024-03-29T04:58:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:42 smithi181 ceph-mon[101970]: osdmap e375: 8 total, 8 up, 8 in 2024-03-29T04:58:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:42 smithi181 ceph-mon[101970]: failed to encode map e375 with expected crc 2024-03-29T04:58:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:42 smithi181 ceph-mon[101970]: failed to encode map e375 with expected crc 2024-03-29T04:58:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:42 smithi181 ceph-mon[101970]: failed to encode map e375 with expected crc 2024-03-29T04:58:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:42 smithi181 ceph-mon[101970]: pgmap v216: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:43.608 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestGroup.add_snapshotPP (6110 ms) 2024-03-29T04:58:43.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:43 smithi187 ceph-mon[93536]: failed to encode map e376 with expected crc 2024-03-29T04:58:43.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:43 smithi187 ceph-mon[93536]: failed to encode map e376 with expected crc 2024-03-29T04:58:43.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:43 smithi187 ceph-mon[93536]: osdmap e376: 8 total, 8 up, 8 in 2024-03-29T04:58:43.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:43 smithi187 ceph-mon[93536]: failed to encode map e376 with expected crc 2024-03-29T04:58:43.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:43 smithi181 ceph-mon[101970]: failed to encode map e376 with expected crc 2024-03-29T04:58:43.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:43 smithi181 ceph-mon[101970]: failed to encode map e376 with expected crc 2024-03-29T04:58:43.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:43 smithi181 ceph-mon[101970]: osdmap e376: 8 total, 8 up, 8 in 2024-03-29T04:58:43.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:43 smithi181 ceph-mon[101970]: failed to encode map e376 with expected crc 2024-03-29T04:58:43.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:43 smithi181 ceph-mon[106466]: failed to encode map e376 with expected crc 2024-03-29T04:58:43.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:43 smithi181 ceph-mon[106466]: failed to encode map e376 with expected crc 2024-03-29T04:58:43.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:43 smithi181 ceph-mon[106466]: osdmap e376: 8 total, 8 up, 8 in 2024-03-29T04:58:43.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:43 smithi181 ceph-mon[106466]: failed to encode map e376 with expected crc 2024-03-29T04:58:44.587 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 6 tests from TestGroup (14143 ms total) 2024-03-29T04:58:44.587 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T04:58:44.587 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 31 tests from TestDeepCopy 2024-03-29T04:58:44.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:44 smithi187 ceph-mon[93536]: failed to encode map e377 with expected crc 2024-03-29T04:58:44.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:44 smithi187 ceph-mon[93536]: failed to encode map e377 with expected crc 2024-03-29T04:58:44.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:44 smithi187 ceph-mon[93536]: osdmap e377: 8 total, 8 up, 8 in 2024-03-29T04:58:44.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:44 smithi187 ceph-mon[93536]: failed to encode map e377 with expected crc 2024-03-29T04:58:44.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:44 smithi187 ceph-mon[93536]: failed to encode map e377 with expected crc 2024-03-29T04:58:44.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:44 smithi187 ceph-mon[93536]: failed to encode map e377 with expected crc 2024-03-29T04:58:44.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:44 smithi187 ceph-mon[93536]: pgmap v219: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 105 KiB/s rd, 10 KiB/s wr, 141 op/s 2024-03-29T04:58:44.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:44 smithi181 ceph-mon[101970]: failed to encode map e377 with expected crc 2024-03-29T04:58:44.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:44 smithi181 ceph-mon[101970]: failed to encode map e377 with expected crc 2024-03-29T04:58:44.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:44 smithi181 ceph-mon[101970]: osdmap e377: 8 total, 8 up, 8 in 2024-03-29T04:58:44.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:44 smithi181 ceph-mon[101970]: failed to encode map e377 with expected crc 2024-03-29T04:58:44.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:44 smithi181 ceph-mon[101970]: failed to encode map e377 with expected crc 2024-03-29T04:58:44.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:44 smithi181 ceph-mon[101970]: failed to encode map e377 with expected crc 2024-03-29T04:58:44.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:44 smithi181 ceph-mon[101970]: pgmap v219: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 105 KiB/s rd, 10 KiB/s wr, 141 op/s 2024-03-29T04:58:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:44 smithi181 ceph-mon[106466]: failed to encode map e377 with expected crc 2024-03-29T04:58:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:44 smithi181 ceph-mon[106466]: failed to encode map e377 with expected crc 2024-03-29T04:58:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:44 smithi181 ceph-mon[106466]: osdmap e377: 8 total, 8 up, 8 in 2024-03-29T04:58:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:44 smithi181 ceph-mon[106466]: failed to encode map e377 with expected crc 2024-03-29T04:58:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:44 smithi181 ceph-mon[106466]: failed to encode map e377 with expected crc 2024-03-29T04:58:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:44 smithi181 ceph-mon[106466]: failed to encode map e377 with expected crc 2024-03-29T04:58:44.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:44 smithi181 ceph-mon[106466]: pgmap v219: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 105 KiB/s rd, 10 KiB/s wr, 141 op/s 2024-03-29T04:58:45.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:45 smithi187 ceph-mon[93536]: failed to encode map e378 with expected crc 2024-03-29T04:58:45.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:45 smithi187 ceph-mon[93536]: failed to encode map e378 with expected crc 2024-03-29T04:58:45.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:45 smithi187 ceph-mon[93536]: osdmap e378: 8 total, 8 up, 8 in 2024-03-29T04:58:45.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:45 smithi181 ceph-mon[101970]: failed to encode map e378 with expected crc 2024-03-29T04:58:45.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:45 smithi181 ceph-mon[101970]: failed to encode map e378 with expected crc 2024-03-29T04:58:45.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:45 smithi181 ceph-mon[101970]: osdmap e378: 8 total, 8 up, 8 in 2024-03-29T04:58:45.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:45 smithi181 ceph-mon[106466]: failed to encode map e378 with expected crc 2024-03-29T04:58:45.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:45 smithi181 ceph-mon[106466]: failed to encode map e378 with expected crc 2024-03-29T04:58:45.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:45 smithi181 ceph-mon[106466]: osdmap e378: 8 total, 8 up, 8 in 2024-03-29T04:58:46.606 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.Empty 2024-03-29T04:58:46.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:46 smithi187 ceph-mon[93536]: failed to encode map e379 with expected crc 2024-03-29T04:58:46.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:46 smithi187 ceph-mon[93536]: failed to encode map e379 with expected crc 2024-03-29T04:58:46.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:46 smithi187 ceph-mon[93536]: osdmap e379: 8 total, 8 up, 8 in 2024-03-29T04:58:46.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:46 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/234465424' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:58:46.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:46 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:58:46.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:46 smithi187 ceph-mon[93536]: pgmap v222: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:46.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:46 smithi181 ceph-mon[101970]: failed to encode map e379 with expected crc 2024-03-29T04:58:46.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:46 smithi181 ceph-mon[101970]: failed to encode map e379 with expected crc 2024-03-29T04:58:46.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:46 smithi181 ceph-mon[101970]: osdmap e379: 8 total, 8 up, 8 in 2024-03-29T04:58:46.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:46 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/234465424' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:58:46.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:46 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:58:46.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:46 smithi181 ceph-mon[101970]: pgmap v222: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:46.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:46 smithi181 ceph-mon[106466]: failed to encode map e379 with expected crc 2024-03-29T04:58:46.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:46 smithi181 ceph-mon[106466]: failed to encode map e379 with expected crc 2024-03-29T04:58:46.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:46 smithi181 ceph-mon[106466]: osdmap e379: 8 total, 8 up, 8 in 2024-03-29T04:58:46.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:46 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/234465424' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:58:46.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:46 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:58:46.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:46 smithi181 ceph-mon[106466]: pgmap v222: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:47.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:47 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:58:47.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:47 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:58:47.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:47 smithi187 ceph-mon[93536]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:47 smithi187 ceph-mon[93536]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:47 smithi187 ceph-mon[93536]: osdmap e380: 8 total, 8 up, 8 in 2024-03-29T04:58:47.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:47 smithi187 ceph-mon[93536]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:47 smithi187 ceph-mon[93536]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:47 smithi187 ceph-mon[93536]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:47 smithi187 ceph-mon[93536]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:47 smithi187 ceph-mon[93536]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:47 smithi187 ceph-mon[93536]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:47 smithi187 ceph-mon[93536]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:47 smithi187 ceph-mon[93536]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:58:47.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:58:47.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[106466]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[106466]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[106466]: osdmap e380: 8 total, 8 up, 8 in 2024-03-29T04:58:47.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[106466]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[106466]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[106466]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[106466]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[106466]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[106466]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[106466]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[106466]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:58:47.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:58:47.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[101970]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[101970]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[101970]: osdmap e380: 8 total, 8 up, 8 in 2024-03-29T04:58:47.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[101970]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[101970]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[101970]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[101970]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[101970]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[101970]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[101970]: failed to encode map e380 with expected crc 2024-03-29T04:58:47.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:47 smithi181 ceph-mon[101970]: failed to encode map e380 with expected crc 2024-03-29T04:58:48.741 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.Empty (2136 ms) 2024-03-29T04:58:48.741 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.NoSnaps 2024-03-29T04:58:48.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:48 smithi187 ceph-mon[93536]: failed to encode map e381 with expected crc 2024-03-29T04:58:48.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:48 smithi187 ceph-mon[93536]: failed to encode map e381 with expected crc 2024-03-29T04:58:48.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:48 smithi187 ceph-mon[93536]: osdmap e381: 8 total, 8 up, 8 in 2024-03-29T04:58:48.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:48 smithi187 ceph-mon[93536]: failed to encode map e381 with expected crc 2024-03-29T04:58:48.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:48 smithi187 ceph-mon[93536]: pgmap v225: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:48.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:48 smithi187 ceph-mon[93536]: failed to encode map e381 with expected crc 2024-03-29T04:58:48.884 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:58:48 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:58:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:58:48.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:48 smithi181 ceph-mon[101970]: failed to encode map e381 with expected crc 2024-03-29T04:58:48.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:48 smithi181 ceph-mon[101970]: failed to encode map e381 with expected crc 2024-03-29T04:58:48.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:48 smithi181 ceph-mon[101970]: osdmap e381: 8 total, 8 up, 8 in 2024-03-29T04:58:48.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:48 smithi181 ceph-mon[101970]: failed to encode map e381 with expected crc 2024-03-29T04:58:48.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:48 smithi181 ceph-mon[101970]: pgmap v225: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:48.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:48 smithi181 ceph-mon[101970]: failed to encode map e381 with expected crc 2024-03-29T04:58:48.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:48 smithi181 ceph-mon[106466]: failed to encode map e381 with expected crc 2024-03-29T04:58:48.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:48 smithi181 ceph-mon[106466]: failed to encode map e381 with expected crc 2024-03-29T04:58:48.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:48 smithi181 ceph-mon[106466]: osdmap e381: 8 total, 8 up, 8 in 2024-03-29T04:58:48.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:48 smithi181 ceph-mon[106466]: failed to encode map e381 with expected crc 2024-03-29T04:58:48.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:48 smithi181 ceph-mon[106466]: pgmap v225: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:48.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:48 smithi181 ceph-mon[106466]: failed to encode map e381 with expected crc 2024-03-29T04:58:48.985 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.NoSnaps (243 ms) 2024-03-29T04:58:48.985 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.Snaps 2024-03-29T04:58:49.453 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:58:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:58:48] "GET /metrics HTTP/1.1" 200 36197 "" "Prometheus/2.43.0" 2024-03-29T04:58:49.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:49 smithi187 ceph-mon[93536]: failed to encode map e381 with expected crc 2024-03-29T04:58:49.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:49 smithi187 ceph-mon[93536]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:58:49.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:49 smithi187 ceph-mon[93536]: failed to encode map e382 with expected crc 2024-03-29T04:58:49.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:49 smithi187 ceph-mon[93536]: failed to encode map e382 with expected crc 2024-03-29T04:58:49.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:49 smithi187 ceph-mon[93536]: osdmap e382: 8 total, 8 up, 8 in 2024-03-29T04:58:49.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:49 smithi187 ceph-mon[93536]: failed to encode map e382 with expected crc 2024-03-29T04:58:49.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:49 smithi187 ceph-mon[93536]: failed to encode map e382 with expected crc 2024-03-29T04:58:49.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:49 smithi187 ceph-mon[93536]: failed to encode map e382 with expected crc 2024-03-29T04:58:49.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:49 smithi187 ceph-mon[93536]: failed to encode map e381 with expected crc 2024-03-29T04:58:49.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[101970]: failed to encode map e381 with expected crc 2024-03-29T04:58:49.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[101970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:58:49.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[101970]: failed to encode map e382 with expected crc 2024-03-29T04:58:49.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[101970]: failed to encode map e382 with expected crc 2024-03-29T04:58:49.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[101970]: osdmap e382: 8 total, 8 up, 8 in 2024-03-29T04:58:49.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[101970]: failed to encode map e382 with expected crc 2024-03-29T04:58:49.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[101970]: failed to encode map e382 with expected crc 2024-03-29T04:58:49.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[101970]: failed to encode map e382 with expected crc 2024-03-29T04:58:49.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[101970]: failed to encode map e381 with expected crc 2024-03-29T04:58:49.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[106466]: failed to encode map e381 with expected crc 2024-03-29T04:58:49.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[106466]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:58:49.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[106466]: failed to encode map e382 with expected crc 2024-03-29T04:58:49.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[106466]: failed to encode map e382 with expected crc 2024-03-29T04:58:49.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[106466]: osdmap e382: 8 total, 8 up, 8 in 2024-03-29T04:58:49.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[106466]: failed to encode map e382 with expected crc 2024-03-29T04:58:49.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[106466]: failed to encode map e382 with expected crc 2024-03-29T04:58:49.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[106466]: failed to encode map e382 with expected crc 2024-03-29T04:58:49.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:49 smithi181 ceph-mon[106466]: failed to encode map e381 with expected crc 2024-03-29T04:58:50.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:50 smithi187 ceph-mon[93536]: failed to encode map e383 with expected crc 2024-03-29T04:58:50.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:50 smithi187 ceph-mon[93536]: failed to encode map e383 with expected crc 2024-03-29T04:58:50.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:50 smithi187 ceph-mon[93536]: osdmap e383: 8 total, 8 up, 8 in 2024-03-29T04:58:50.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:50 smithi187 ceph-mon[93536]: failed to encode map e383 with expected crc 2024-03-29T04:58:50.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:50 smithi187 ceph-mon[93536]: failed to encode map e383 with expected crc 2024-03-29T04:58:50.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:50 smithi187 ceph-mon[93536]: pgmap v228: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:50.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:50 smithi181 ceph-mon[106466]: failed to encode map e383 with expected crc 2024-03-29T04:58:50.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:50 smithi181 ceph-mon[106466]: failed to encode map e383 with expected crc 2024-03-29T04:58:50.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:50 smithi181 ceph-mon[106466]: osdmap e383: 8 total, 8 up, 8 in 2024-03-29T04:58:50.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:50 smithi181 ceph-mon[106466]: failed to encode map e383 with expected crc 2024-03-29T04:58:50.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:50 smithi181 ceph-mon[106466]: failed to encode map e383 with expected crc 2024-03-29T04:58:50.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:50 smithi181 ceph-mon[106466]: pgmap v228: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:50.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:50 smithi181 ceph-mon[101970]: failed to encode map e383 with expected crc 2024-03-29T04:58:50.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:50 smithi181 ceph-mon[101970]: failed to encode map e383 with expected crc 2024-03-29T04:58:50.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:50 smithi181 ceph-mon[101970]: osdmap e383: 8 total, 8 up, 8 in 2024-03-29T04:58:50.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:50 smithi181 ceph-mon[101970]: failed to encode map e383 with expected crc 2024-03-29T04:58:50.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:50 smithi181 ceph-mon[101970]: failed to encode map e383 with expected crc 2024-03-29T04:58:50.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:50 smithi181 ceph-mon[101970]: pgmap v228: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:51.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:51 smithi187 ceph-mon[93536]: failed to encode map e384 with expected crc 2024-03-29T04:58:51.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:51 smithi187 ceph-mon[93536]: failed to encode map e384 with expected crc 2024-03-29T04:58:51.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:51 smithi187 ceph-mon[93536]: osdmap e384: 8 total, 8 up, 8 in 2024-03-29T04:58:51.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:51 smithi187 ceph-mon[93536]: failed to encode map e384 with expected crc 2024-03-29T04:58:51.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:51 smithi181 ceph-mon[106466]: failed to encode map e384 with expected crc 2024-03-29T04:58:51.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:51 smithi181 ceph-mon[106466]: failed to encode map e384 with expected crc 2024-03-29T04:58:51.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:51 smithi181 ceph-mon[106466]: osdmap e384: 8 total, 8 up, 8 in 2024-03-29T04:58:51.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:51 smithi181 ceph-mon[106466]: failed to encode map e384 with expected crc 2024-03-29T04:58:51.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:51 smithi181 ceph-mon[101970]: failed to encode map e384 with expected crc 2024-03-29T04:58:51.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:51 smithi181 ceph-mon[101970]: failed to encode map e384 with expected crc 2024-03-29T04:58:51.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:51 smithi181 ceph-mon[101970]: osdmap e384: 8 total, 8 up, 8 in 2024-03-29T04:58:51.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:51 smithi181 ceph-mon[101970]: failed to encode map e384 with expected crc 2024-03-29T04:58:52.925 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.Snaps (3940 ms) 2024-03-29T04:58:52.925 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.SnapDiscard 2024-03-29T04:58:52.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:52 smithi181 ceph-mon[101970]: failed to encode map e385 with expected crc 2024-03-29T04:58:52.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:52 smithi181 ceph-mon[101970]: failed to encode map e385 with expected crc 2024-03-29T04:58:52.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:52 smithi181 ceph-mon[101970]: osdmap e385: 8 total, 8 up, 8 in 2024-03-29T04:58:52.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:52 smithi181 ceph-mon[101970]: failed to encode map e385 with expected crc 2024-03-29T04:58:52.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:52 smithi181 ceph-mon[101970]: pgmap v231: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:52.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:52 smithi181 ceph-mon[106466]: failed to encode map e385 with expected crc 2024-03-29T04:58:52.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:52 smithi181 ceph-mon[106466]: failed to encode map e385 with expected crc 2024-03-29T04:58:52.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:52 smithi181 ceph-mon[106466]: osdmap e385: 8 total, 8 up, 8 in 2024-03-29T04:58:52.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:52 smithi181 ceph-mon[106466]: failed to encode map e385 with expected crc 2024-03-29T04:58:52.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:52 smithi181 ceph-mon[106466]: pgmap v231: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:53.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:52 smithi187 ceph-mon[93536]: failed to encode map e385 with expected crc 2024-03-29T04:58:53.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:52 smithi187 ceph-mon[93536]: failed to encode map e385 with expected crc 2024-03-29T04:58:53.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:52 smithi187 ceph-mon[93536]: osdmap e385: 8 total, 8 up, 8 in 2024-03-29T04:58:53.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:52 smithi187 ceph-mon[93536]: failed to encode map e385 with expected crc 2024-03-29T04:58:53.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:52 smithi187 ceph-mon[93536]: pgmap v231: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:58:53.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:53 smithi181 ceph-mon[106466]: failed to encode map e386 with expected crc 2024-03-29T04:58:53.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:53 smithi181 ceph-mon[106466]: failed to encode map e386 with expected crc 2024-03-29T04:58:53.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:53 smithi181 ceph-mon[106466]: osdmap e386: 8 total, 8 up, 8 in 2024-03-29T04:58:53.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:53 smithi181 ceph-mon[106466]: failed to encode map e386 with expected crc 2024-03-29T04:58:53.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:53 smithi181 ceph-mon[106466]: failed to encode map e386 with expected crc 2024-03-29T04:58:53.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:53 smithi181 ceph-mon[106466]: failed to encode map e386 with expected crc 2024-03-29T04:58:53.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:53 smithi181 ceph-mon[106466]: failed to encode map e386 with expected crc 2024-03-29T04:58:53.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:53 smithi181 ceph-mon[101970]: failed to encode map e386 with expected crc 2024-03-29T04:58:53.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:53 smithi181 ceph-mon[101970]: failed to encode map e386 with expected crc 2024-03-29T04:58:53.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:53 smithi181 ceph-mon[101970]: osdmap e386: 8 total, 8 up, 8 in 2024-03-29T04:58:53.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:53 smithi181 ceph-mon[101970]: failed to encode map e386 with expected crc 2024-03-29T04:58:53.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:53 smithi181 ceph-mon[101970]: failed to encode map e386 with expected crc 2024-03-29T04:58:53.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:53 smithi181 ceph-mon[101970]: failed to encode map e386 with expected crc 2024-03-29T04:58:53.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:53 smithi181 ceph-mon[101970]: failed to encode map e386 with expected crc 2024-03-29T04:58:54.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:53 smithi187 ceph-mon[93536]: failed to encode map e386 with expected crc 2024-03-29T04:58:54.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:53 smithi187 ceph-mon[93536]: failed to encode map e386 with expected crc 2024-03-29T04:58:54.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:53 smithi187 ceph-mon[93536]: osdmap e386: 8 total, 8 up, 8 in 2024-03-29T04:58:54.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:53 smithi187 ceph-mon[93536]: failed to encode map e386 with expected crc 2024-03-29T04:58:54.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:53 smithi187 ceph-mon[93536]: failed to encode map e386 with expected crc 2024-03-29T04:58:54.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:53 smithi187 ceph-mon[93536]: failed to encode map e386 with expected crc 2024-03-29T04:58:54.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:53 smithi187 ceph-mon[93536]: failed to encode map e386 with expected crc 2024-03-29T04:58:54.772 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.SnapDiscard (1846 ms) 2024-03-29T04:58:54.772 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.CloneDiscard 2024-03-29T04:58:54.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:54 smithi181 ceph-mon[101970]: failed to encode map e387 with expected crc 2024-03-29T04:58:54.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:54 smithi181 ceph-mon[101970]: failed to encode map e387 with expected crc 2024-03-29T04:58:54.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:54 smithi181 ceph-mon[101970]: osdmap e387: 8 total, 8 up, 8 in 2024-03-29T04:58:54.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:54 smithi181 ceph-mon[101970]: failed to encode map e387 with expected crc 2024-03-29T04:58:54.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:54 smithi181 ceph-mon[101970]: pgmap v234: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 176 op/s 2024-03-29T04:58:54.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:54 smithi181 ceph-mon[106466]: failed to encode map e387 with expected crc 2024-03-29T04:58:54.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:54 smithi181 ceph-mon[106466]: failed to encode map e387 with expected crc 2024-03-29T04:58:54.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:54 smithi181 ceph-mon[106466]: osdmap e387: 8 total, 8 up, 8 in 2024-03-29T04:58:54.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:54 smithi181 ceph-mon[106466]: failed to encode map e387 with expected crc 2024-03-29T04:58:54.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:54 smithi181 ceph-mon[106466]: pgmap v234: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 176 op/s 2024-03-29T04:58:55.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:54 smithi187 ceph-mon[93536]: failed to encode map e387 with expected crc 2024-03-29T04:58:55.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:54 smithi187 ceph-mon[93536]: failed to encode map e387 with expected crc 2024-03-29T04:58:55.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:54 smithi187 ceph-mon[93536]: osdmap e387: 8 total, 8 up, 8 in 2024-03-29T04:58:55.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:54 smithi187 ceph-mon[93536]: failed to encode map e387 with expected crc 2024-03-29T04:58:55.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:54 smithi187 ceph-mon[93536]: pgmap v234: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 176 op/s 2024-03-29T04:58:55.851 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.CloneDiscard (1081 ms) 2024-03-29T04:58:55.852 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.CloneShrink 2024-03-29T04:58:55.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:55 smithi181 ceph-mon[101970]: failed to encode map e388 with expected crc 2024-03-29T04:58:55.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:55 smithi181 ceph-mon[101970]: failed to encode map e388 with expected crc 2024-03-29T04:58:55.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:55 smithi181 ceph-mon[101970]: osdmap e388: 8 total, 8 up, 8 in 2024-03-29T04:58:55.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:55 smithi181 ceph-mon[101970]: failed to encode map e388 with expected crc 2024-03-29T04:58:55.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:55 smithi181 ceph-mon[101970]: failed to encode map e388 with expected crc 2024-03-29T04:58:55.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:55 smithi181 ceph-mon[106466]: failed to encode map e388 with expected crc 2024-03-29T04:58:55.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:55 smithi181 ceph-mon[106466]: failed to encode map e388 with expected crc 2024-03-29T04:58:55.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:55 smithi181 ceph-mon[106466]: osdmap e388: 8 total, 8 up, 8 in 2024-03-29T04:58:55.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:55 smithi181 ceph-mon[106466]: failed to encode map e388 with expected crc 2024-03-29T04:58:55.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:55 smithi181 ceph-mon[106466]: failed to encode map e388 with expected crc 2024-03-29T04:58:56.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:55 smithi187 ceph-mon[93536]: failed to encode map e388 with expected crc 2024-03-29T04:58:56.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:55 smithi187 ceph-mon[93536]: failed to encode map e388 with expected crc 2024-03-29T04:58:56.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:55 smithi187 ceph-mon[93536]: osdmap e388: 8 total, 8 up, 8 in 2024-03-29T04:58:56.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:55 smithi187 ceph-mon[93536]: failed to encode map e388 with expected crc 2024-03-29T04:58:56.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:55 smithi187 ceph-mon[93536]: failed to encode map e388 with expected crc 2024-03-29T04:58:56.846 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.CloneShrink (995 ms) 2024-03-29T04:58:56.846 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.CloneExpand 2024-03-29T04:58:56.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:56 smithi181 ceph-mon[101970]: failed to encode map e389 with expected crc 2024-03-29T04:58:56.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:56 smithi181 ceph-mon[101970]: failed to encode map e389 with expected crc 2024-03-29T04:58:56.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:56 smithi181 ceph-mon[101970]: osdmap e389: 8 total, 8 up, 8 in 2024-03-29T04:58:56.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:56 smithi181 ceph-mon[101970]: failed to encode map e389 with expected crc 2024-03-29T04:58:56.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:56 smithi181 ceph-mon[101970]: failed to encode map e389 with expected crc 2024-03-29T04:58:56.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:56 smithi181 ceph-mon[101970]: pgmap v237: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 176 op/s 2024-03-29T04:58:56.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:56.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:56.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:56 smithi181 ceph-mon[106466]: failed to encode map e389 with expected crc 2024-03-29T04:58:56.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:56 smithi181 ceph-mon[106466]: failed to encode map e389 with expected crc 2024-03-29T04:58:56.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:56 smithi181 ceph-mon[106466]: osdmap e389: 8 total, 8 up, 8 in 2024-03-29T04:58:56.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:56 smithi181 ceph-mon[106466]: failed to encode map e389 with expected crc 2024-03-29T04:58:56.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:56 smithi181 ceph-mon[106466]: failed to encode map e389 with expected crc 2024-03-29T04:58:56.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:56 smithi181 ceph-mon[106466]: pgmap v237: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 176 op/s 2024-03-29T04:58:56.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:56.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:57.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:56 smithi187 ceph-mon[93536]: failed to encode map e389 with expected crc 2024-03-29T04:58:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:56 smithi187 ceph-mon[93536]: failed to encode map e389 with expected crc 2024-03-29T04:58:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:56 smithi187 ceph-mon[93536]: osdmap e389: 8 total, 8 up, 8 in 2024-03-29T04:58:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:56 smithi187 ceph-mon[93536]: failed to encode map e389 with expected crc 2024-03-29T04:58:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:56 smithi187 ceph-mon[93536]: failed to encode map e389 with expected crc 2024-03-29T04:58:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:56 smithi187 ceph-mon[93536]: pgmap v237: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 176 op/s 2024-03-29T04:58:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:58:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:58:57.697 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.CloneExpand (850 ms) 2024-03-29T04:58:57.698 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.CloneHideParent 2024-03-29T04:58:57.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[101970]: failed to encode map e390 with expected crc 2024-03-29T04:58:57.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[101970]: failed to encode map e390 with expected crc 2024-03-29T04:58:57.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[101970]: osdmap e390: 8 total, 8 up, 8 in 2024-03-29T04:58:57.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[101970]: failed to encode map e390 with expected crc 2024-03-29T04:58:57.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[101970]: failed to encode map e390 with expected crc 2024-03-29T04:58:57.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[101970]: failed to encode map e391 with expected crc 2024-03-29T04:58:57.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[101970]: osdmap e391: 8 total, 8 up, 8 in 2024-03-29T04:58:57.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[101970]: failed to encode map e391 with expected crc 2024-03-29T04:58:57.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[101970]: failed to encode map e391 with expected crc 2024-03-29T04:58:57.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[106466]: failed to encode map e390 with expected crc 2024-03-29T04:58:57.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[106466]: failed to encode map e390 with expected crc 2024-03-29T04:58:57.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[106466]: osdmap e390: 8 total, 8 up, 8 in 2024-03-29T04:58:57.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[106466]: failed to encode map e390 with expected crc 2024-03-29T04:58:57.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[106466]: failed to encode map e390 with expected crc 2024-03-29T04:58:57.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[106466]: failed to encode map e391 with expected crc 2024-03-29T04:58:57.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[106466]: osdmap e391: 8 total, 8 up, 8 in 2024-03-29T04:58:57.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[106466]: failed to encode map e391 with expected crc 2024-03-29T04:58:57.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:57 smithi181 ceph-mon[106466]: failed to encode map e391 with expected crc 2024-03-29T04:58:58.035 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:57 smithi187 ceph-mon[93536]: failed to encode map e390 with expected crc 2024-03-29T04:58:58.035 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:57 smithi187 ceph-mon[93536]: failed to encode map e390 with expected crc 2024-03-29T04:58:58.035 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:57 smithi187 ceph-mon[93536]: osdmap e390: 8 total, 8 up, 8 in 2024-03-29T04:58:58.035 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:57 smithi187 ceph-mon[93536]: failed to encode map e390 with expected crc 2024-03-29T04:58:58.035 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:57 smithi187 ceph-mon[93536]: failed to encode map e390 with expected crc 2024-03-29T04:58:58.035 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:57 smithi187 ceph-mon[93536]: failed to encode map e391 with expected crc 2024-03-29T04:58:58.035 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:57 smithi187 ceph-mon[93536]: osdmap e391: 8 total, 8 up, 8 in 2024-03-29T04:58:58.035 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:57 smithi187 ceph-mon[93536]: failed to encode map e391 with expected crc 2024-03-29T04:58:58.036 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:57 smithi187 ceph-mon[93536]: failed to encode map e391 with expected crc 2024-03-29T04:58:58.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:58 smithi181 ceph-mon[106466]: failed to encode map e391 with expected crc 2024-03-29T04:58:58.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:58 smithi181 ceph-mon[106466]: failed to encode map e391 with expected crc 2024-03-29T04:58:58.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:58 smithi181 ceph-mon[106466]: pgmap v240: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 437 op/s 2024-03-29T04:58:58.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:58 smithi181 ceph-mon[101970]: failed to encode map e391 with expected crc 2024-03-29T04:58:58.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:58 smithi181 ceph-mon[101970]: failed to encode map e391 with expected crc 2024-03-29T04:58:58.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:58 smithi181 ceph-mon[101970]: pgmap v240: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 437 op/s 2024-03-29T04:58:59.051 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:58 smithi187 ceph-mon[93536]: failed to encode map e391 with expected crc 2024-03-29T04:58:59.051 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:58 smithi187 ceph-mon[93536]: failed to encode map e391 with expected crc 2024-03-29T04:58:59.051 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:58 smithi187 ceph-mon[93536]: pgmap v240: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 437 op/s 2024-03-29T04:58:59.051 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:58:58 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:58:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:58:59.203 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:58:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:58:58] "GET /metrics HTTP/1.1" 200 36288 "" "Prometheus/2.43.0" 2024-03-29T04:58:59.818 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade resume' 2024-03-29T04:58:59.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:59 smithi181 ceph-mon[101970]: failed to encode map e392 with expected crc 2024-03-29T04:58:59.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:59 smithi181 ceph-mon[101970]: failed to encode map e392 with expected crc 2024-03-29T04:58:59.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:59 smithi181 ceph-mon[101970]: osdmap e392: 8 total, 8 up, 8 in 2024-03-29T04:58:59.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:59 smithi181 ceph-mon[101970]: failed to encode map e392 with expected crc 2024-03-29T04:58:59.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:59 smithi181 ceph-mon[101970]: failed to encode map e392 with expected crc 2024-03-29T04:58:59.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:58:59 smithi181 ceph-mon[101970]: failed to encode map e392 with expected crc 2024-03-29T04:58:59.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:59 smithi181 ceph-mon[106466]: failed to encode map e392 with expected crc 2024-03-29T04:58:59.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:59 smithi181 ceph-mon[106466]: failed to encode map e392 with expected crc 2024-03-29T04:58:59.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:59 smithi181 ceph-mon[106466]: osdmap e392: 8 total, 8 up, 8 in 2024-03-29T04:58:59.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:59 smithi181 ceph-mon[106466]: failed to encode map e392 with expected crc 2024-03-29T04:58:59.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:59 smithi181 ceph-mon[106466]: failed to encode map e392 with expected crc 2024-03-29T04:58:59.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:58:59 smithi181 ceph-mon[106466]: failed to encode map e392 with expected crc 2024-03-29T04:59:00.079 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:59:00.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:59 smithi187 ceph-mon[93536]: failed to encode map e392 with expected crc 2024-03-29T04:59:00.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:59 smithi187 ceph-mon[93536]: failed to encode map e392 with expected crc 2024-03-29T04:59:00.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:59 smithi187 ceph-mon[93536]: osdmap e392: 8 total, 8 up, 8 in 2024-03-29T04:59:00.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:59 smithi187 ceph-mon[93536]: failed to encode map e392 with expected crc 2024-03-29T04:59:00.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:59 smithi187 ceph-mon[93536]: failed to encode map e392 with expected crc 2024-03-29T04:59:00.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:58:59 smithi187 ceph-mon[93536]: failed to encode map e392 with expected crc 2024-03-29T04:59:00.788 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:00 smithi181 ceph-mon[106466]: failed to encode map e393 with expected crc 2024-03-29T04:59:00.788 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:00 smithi181 ceph-mon[106466]: failed to encode map e393 with expected crc 2024-03-29T04:59:00.788 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:00 smithi181 ceph-mon[106466]: osdmap e393: 8 total, 8 up, 8 in 2024-03-29T04:59:00.788 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:00 smithi181 ceph-mon[106466]: failed to encode map e393 with expected crc 2024-03-29T04:59:00.788 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:00 smithi181 ceph-mon[106466]: failed to encode map e393 with expected crc 2024-03-29T04:59:00.788 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:00 smithi181 ceph-mon[106466]: pgmap v243: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 437 op/s 2024-03-29T04:59:00.789 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:00 smithi181 ceph-mon[101970]: failed to encode map e393 with expected crc 2024-03-29T04:59:00.789 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:00 smithi181 ceph-mon[101970]: failed to encode map e393 with expected crc 2024-03-29T04:59:00.789 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:00 smithi181 ceph-mon[101970]: osdmap e393: 8 total, 8 up, 8 in 2024-03-29T04:59:00.789 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:00 smithi181 ceph-mon[101970]: failed to encode map e393 with expected crc 2024-03-29T04:59:00.789 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:00 smithi181 ceph-mon[101970]: failed to encode map e393 with expected crc 2024-03-29T04:59:00.789 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:00 smithi181 ceph-mon[101970]: pgmap v243: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 437 op/s 2024-03-29T04:59:01.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:00 smithi187 ceph-mon[93536]: failed to encode map e393 with expected crc 2024-03-29T04:59:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:00 smithi187 ceph-mon[93536]: failed to encode map e393 with expected crc 2024-03-29T04:59:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:00 smithi187 ceph-mon[93536]: osdmap e393: 8 total, 8 up, 8 in 2024-03-29T04:59:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:00 smithi187 ceph-mon[93536]: failed to encode map e393 with expected crc 2024-03-29T04:59:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:00 smithi187 ceph-mon[93536]: failed to encode map e393 with expected crc 2024-03-29T04:59:01.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:00 smithi187 ceph-mon[93536]: pgmap v243: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 437 op/s 2024-03-29T04:59:01.734 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:01 smithi181 ceph-mon[106466]: failed to encode map e394 with expected crc 2024-03-29T04:59:01.734 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:01 smithi181 ceph-mon[106466]: failed to encode map e394 with expected crc 2024-03-29T04:59:01.735 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:01 smithi181 ceph-mon[106466]: osdmap e394: 8 total, 8 up, 8 in 2024-03-29T04:59:01.735 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:01 smithi181 ceph-mon[106466]: failed to encode map e394 with expected crc 2024-03-29T04:59:01.735 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:01 smithi181 ceph-mon[106466]: failed to encode map e394 with expected crc 2024-03-29T04:59:01.735 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:01 smithi181 ceph-mon[106466]: failed to encode map e394 with expected crc 2024-03-29T04:59:01.735 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:01 smithi181 ceph-mon[106466]: failed to encode map e394 with expected crc 2024-03-29T04:59:01.736 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:01 smithi181 ceph-mon[101970]: failed to encode map e394 with expected crc 2024-03-29T04:59:01.736 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:01 smithi181 ceph-mon[101970]: failed to encode map e394 with expected crc 2024-03-29T04:59:01.736 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:01 smithi181 ceph-mon[101970]: osdmap e394: 8 total, 8 up, 8 in 2024-03-29T04:59:01.736 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:01 smithi181 ceph-mon[101970]: failed to encode map e394 with expected crc 2024-03-29T04:59:01.736 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:01 smithi181 ceph-mon[101970]: failed to encode map e394 with expected crc 2024-03-29T04:59:01.736 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:01 smithi181 ceph-mon[101970]: failed to encode map e394 with expected crc 2024-03-29T04:59:01.736 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:01 smithi181 ceph-mon[101970]: failed to encode map e394 with expected crc 2024-03-29T04:59:01.968 INFO:teuthology.orchestra.run.smithi181.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:59:02.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:01 smithi187 ceph-mon[93536]: failed to encode map e394 with expected crc 2024-03-29T04:59:02.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:01 smithi187 ceph-mon[93536]: failed to encode map e394 with expected crc 2024-03-29T04:59:02.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:01 smithi187 ceph-mon[93536]: osdmap e394: 8 total, 8 up, 8 in 2024-03-29T04:59:02.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:01 smithi187 ceph-mon[93536]: failed to encode map e394 with expected crc 2024-03-29T04:59:02.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:01 smithi187 ceph-mon[93536]: failed to encode map e394 with expected crc 2024-03-29T04:59:02.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:01 smithi187 ceph-mon[93536]: failed to encode map e394 with expected crc 2024-03-29T04:59:02.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:01 smithi187 ceph-mon[93536]: failed to encode map e394 with expected crc 2024-03-29T04:59:02.636 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'echo wait for all mons to upgrade' 2024-03-29T04:59:02.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[106466]: failed to encode map e395 with expected crc 2024-03-29T04:59:02.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[106466]: failed to encode map e395 with expected crc 2024-03-29T04:59:02.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[106466]: osdmap e395: 8 total, 8 up, 8 in 2024-03-29T04:59:02.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[106466]: failed to encode map e395 with expected crc 2024-03-29T04:59:02.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[106466]: pgmap v246: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:59:02.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[106466]: failed to encode map e395 with expected crc 2024-03-29T04:59:02.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[106466]: failed to encode map e395 with expected crc 2024-03-29T04:59:02.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:02.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:59:02.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[101970]: failed to encode map e395 with expected crc 2024-03-29T04:59:02.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[101970]: failed to encode map e395 with expected crc 2024-03-29T04:59:02.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[101970]: osdmap e395: 8 total, 8 up, 8 in 2024-03-29T04:59:02.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[101970]: failed to encode map e395 with expected crc 2024-03-29T04:59:02.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[101970]: pgmap v246: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:59:02.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[101970]: failed to encode map e395 with expected crc 2024-03-29T04:59:02.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[101970]: failed to encode map e395 with expected crc 2024-03-29T04:59:02.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:02.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:02 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:59:03.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:02 smithi187 ceph-mon[93536]: failed to encode map e395 with expected crc 2024-03-29T04:59:03.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:02 smithi187 ceph-mon[93536]: failed to encode map e395 with expected crc 2024-03-29T04:59:03.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:02 smithi187 ceph-mon[93536]: osdmap e395: 8 total, 8 up, 8 in 2024-03-29T04:59:03.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:02 smithi187 ceph-mon[93536]: failed to encode map e395 with expected crc 2024-03-29T04:59:03.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:02 smithi187 ceph-mon[93536]: pgmap v246: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:59:03.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:02 smithi187 ceph-mon[93536]: failed to encode map e395 with expected crc 2024-03-29T04:59:03.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:02 smithi187 ceph-mon[93536]: failed to encode map e395 with expected crc 2024-03-29T04:59:03.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:02 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:03.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:02 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:59:03.458 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:59:03.839 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:03 smithi181 ceph-mon[106466]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:59:03.839 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:03 smithi181 ceph-mon[106466]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:59:03.839 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:03 smithi181 ceph-mon[106466]: failed to encode map e396 with expected crc 2024-03-29T04:59:03.839 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:03 smithi181 ceph-mon[106466]: failed to encode map e396 with expected crc 2024-03-29T04:59:03.839 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:03 smithi181 ceph-mon[106466]: osdmap e396: 8 total, 8 up, 8 in 2024-03-29T04:59:03.839 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:03 smithi181 ceph-mon[106466]: failed to encode map e396 with expected crc 2024-03-29T04:59:03.840 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:03 smithi181 ceph-mon[101970]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:59:03.840 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:03 smithi181 ceph-mon[101970]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:59:03.840 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:03 smithi181 ceph-mon[101970]: failed to encode map e396 with expected crc 2024-03-29T04:59:03.840 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:03 smithi181 ceph-mon[101970]: failed to encode map e396 with expected crc 2024-03-29T04:59:03.840 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:03 smithi181 ceph-mon[101970]: osdmap e396: 8 total, 8 up, 8 in 2024-03-29T04:59:03.840 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:03 smithi181 ceph-mon[101970]: failed to encode map e396 with expected crc 2024-03-29T04:59:04.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:03 smithi187 ceph-mon[93536]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:59:04.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:03 smithi187 ceph-mon[93536]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:59:04.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:03 smithi187 ceph-mon[93536]: failed to encode map e396 with expected crc 2024-03-29T04:59:04.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:03 smithi187 ceph-mon[93536]: failed to encode map e396 with expected crc 2024-03-29T04:59:04.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:03 smithi187 ceph-mon[93536]: osdmap e396: 8 total, 8 up, 8 in 2024-03-29T04:59:04.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:03 smithi187 ceph-mon[93536]: failed to encode map e396 with expected crc 2024-03-29T04:59:04.821 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.CloneHideParent (7124 ms) 2024-03-29T04:59:04.821 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.Clone 2024-03-29T04:59:04.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:04 smithi181 ceph-mon[106466]: failed to encode map e396 with expected crc 2024-03-29T04:59:04.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:04 smithi181 ceph-mon[106466]: failed to encode map e397 with expected crc 2024-03-29T04:59:04.947 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:04 smithi181 ceph-mon[106466]: failed to encode map e397 with expected crc 2024-03-29T04:59:04.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:04 smithi181 ceph-mon[106466]: osdmap e397: 8 total, 8 up, 8 in 2024-03-29T04:59:04.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:04 smithi181 ceph-mon[106466]: pgmap v249: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-29T04:59:04.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:04 smithi181 ceph-mon[106466]: failed to encode map e397 with expected crc 2024-03-29T04:59:04.948 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:04 smithi181 ceph-mon[106466]: failed to encode map e396 with expected crc 2024-03-29T04:59:04.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:04 smithi181 ceph-mon[101970]: failed to encode map e396 with expected crc 2024-03-29T04:59:04.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:04 smithi181 ceph-mon[101970]: failed to encode map e397 with expected crc 2024-03-29T04:59:04.948 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:04 smithi181 ceph-mon[101970]: failed to encode map e397 with expected crc 2024-03-29T04:59:04.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:04 smithi181 ceph-mon[101970]: osdmap e397: 8 total, 8 up, 8 in 2024-03-29T04:59:04.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:04 smithi181 ceph-mon[101970]: pgmap v249: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-29T04:59:04.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:04 smithi181 ceph-mon[101970]: failed to encode map e397 with expected crc 2024-03-29T04:59:04.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:04 smithi181 ceph-mon[101970]: failed to encode map e396 with expected crc 2024-03-29T04:59:05.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:04 smithi187 ceph-mon[93536]: failed to encode map e396 with expected crc 2024-03-29T04:59:05.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:04 smithi187 ceph-mon[93536]: failed to encode map e397 with expected crc 2024-03-29T04:59:05.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:04 smithi187 ceph-mon[93536]: failed to encode map e397 with expected crc 2024-03-29T04:59:05.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:04 smithi187 ceph-mon[93536]: osdmap e397: 8 total, 8 up, 8 in 2024-03-29T04:59:05.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:04 smithi187 ceph-mon[93536]: pgmap v249: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-29T04:59:05.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:04 smithi187 ceph-mon[93536]: failed to encode map e397 with expected crc 2024-03-29T04:59:05.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:04 smithi187 ceph-mon[93536]: failed to encode map e396 with expected crc 2024-03-29T04:59:05.399 INFO:teuthology.orchestra.run.smithi181.stdout:wait for all mons to upgrade 2024-03-29T04:59:06.054 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:05 smithi181 ceph-mon[106466]: failed to encode map e398 with expected crc 2024-03-29T04:59:06.054 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:05 smithi181 ceph-mon[106466]: failed to encode map e398 with expected crc 2024-03-29T04:59:06.054 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:05 smithi181 ceph-mon[106466]: osdmap e398: 8 total, 8 up, 8 in 2024-03-29T04:59:06.055 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:05 smithi181 ceph-mon[106466]: failed to encode map e398 with expected crc 2024-03-29T04:59:06.055 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:05 smithi181 ceph-mon[101970]: failed to encode map e398 with expected crc 2024-03-29T04:59:06.055 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:05 smithi181 ceph-mon[101970]: failed to encode map e398 with expected crc 2024-03-29T04:59:06.055 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:05 smithi181 ceph-mon[101970]: osdmap e398: 8 total, 8 up, 8 in 2024-03-29T04:59:06.055 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:05 smithi181 ceph-mon[101970]: failed to encode map e398 with expected crc 2024-03-29T04:59:06.079 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-03-29T04:59:06.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:05 smithi187 ceph-mon[93536]: failed to encode map e398 with expected crc 2024-03-29T04:59:06.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:05 smithi187 ceph-mon[93536]: failed to encode map e398 with expected crc 2024-03-29T04:59:06.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:05 smithi187 ceph-mon[93536]: osdmap e398: 8 total, 8 up, 8 in 2024-03-29T04:59:06.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:05 smithi187 ceph-mon[93536]: failed to encode map e398 with expected crc 2024-03-29T04:59:06.572 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:59:06.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:06 smithi181 ceph-mon[101970]: failed to encode map e399 with expected crc 2024-03-29T04:59:06.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:06 smithi181 ceph-mon[101970]: failed to encode map e399 with expected crc 2024-03-29T04:59:06.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:06 smithi181 ceph-mon[101970]: osdmap e399: 8 total, 8 up, 8 in 2024-03-29T04:59:06.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:06 smithi181 ceph-mon[101970]: failed to encode map e399 with expected crc 2024-03-29T04:59:06.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:06 smithi181 ceph-mon[101970]: pgmap v252: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-29T04:59:06.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:06 smithi181 ceph-mon[101970]: failed to encode map e399 with expected crc 2024-03-29T04:59:06.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:06 smithi181 ceph-mon[101970]: failed to encode map e399 with expected crc 2024-03-29T04:59:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:06 smithi181 ceph-mon[106466]: failed to encode map e399 with expected crc 2024-03-29T04:59:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:06 smithi181 ceph-mon[106466]: failed to encode map e399 with expected crc 2024-03-29T04:59:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:06 smithi181 ceph-mon[106466]: osdmap e399: 8 total, 8 up, 8 in 2024-03-29T04:59:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:06 smithi181 ceph-mon[106466]: failed to encode map e399 with expected crc 2024-03-29T04:59:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:06 smithi181 ceph-mon[106466]: pgmap v252: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-29T04:59:06.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:06 smithi181 ceph-mon[106466]: failed to encode map e399 with expected crc 2024-03-29T04:59:06.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:06 smithi181 ceph-mon[106466]: failed to encode map e399 with expected crc 2024-03-29T04:59:07.064 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.Clone (2243 ms) 2024-03-29T04:59:07.064 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.CloneFlatten 2024-03-29T04:59:07.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:06 smithi187 ceph-mon[93536]: failed to encode map e399 with expected crc 2024-03-29T04:59:07.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:06 smithi187 ceph-mon[93536]: failed to encode map e399 with expected crc 2024-03-29T04:59:07.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:06 smithi187 ceph-mon[93536]: osdmap e399: 8 total, 8 up, 8 in 2024-03-29T04:59:07.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:06 smithi187 ceph-mon[93536]: failed to encode map e399 with expected crc 2024-03-29T04:59:07.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:06 smithi187 ceph-mon[93536]: pgmap v252: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-29T04:59:07.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:06 smithi187 ceph-mon[93536]: failed to encode map e399 with expected crc 2024-03-29T04:59:07.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:06 smithi187 ceph-mon[93536]: failed to encode map e399 with expected crc 2024-03-29T04:59:07.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:07 smithi181 ceph-mon[101970]: failed to encode map e400 with expected crc 2024-03-29T04:59:07.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:07 smithi181 ceph-mon[101970]: failed to encode map e400 with expected crc 2024-03-29T04:59:07.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:07 smithi181 ceph-mon[101970]: osdmap e400: 8 total, 8 up, 8 in 2024-03-29T04:59:07.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:07 smithi181 ceph-mon[101970]: failed to encode map e400 with expected crc 2024-03-29T04:59:07.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:07 smithi181 ceph-mon[101970]: failed to encode map e400 with expected crc 2024-03-29T04:59:07.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:07 smithi181 ceph-mon[106466]: failed to encode map e400 with expected crc 2024-03-29T04:59:07.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:07 smithi181 ceph-mon[106466]: failed to encode map e400 with expected crc 2024-03-29T04:59:07.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:07 smithi181 ceph-mon[106466]: osdmap e400: 8 total, 8 up, 8 in 2024-03-29T04:59:07.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:07 smithi181 ceph-mon[106466]: failed to encode map e400 with expected crc 2024-03-29T04:59:07.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:07 smithi181 ceph-mon[106466]: failed to encode map e400 with expected crc 2024-03-29T04:59:08.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:07 smithi187 ceph-mon[93536]: failed to encode map e400 with expected crc 2024-03-29T04:59:08.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:07 smithi187 ceph-mon[93536]: failed to encode map e400 with expected crc 2024-03-29T04:59:08.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:07 smithi187 ceph-mon[93536]: osdmap e400: 8 total, 8 up, 8 in 2024-03-29T04:59:08.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:07 smithi187 ceph-mon[93536]: failed to encode map e400 with expected crc 2024-03-29T04:59:08.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:07 smithi187 ceph-mon[93536]: failed to encode map e400 with expected crc 2024-03-29T04:59:08.574 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 3 2024-03-29T04:59:08.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[101970]: pgmap v254: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 341 op/s 2024-03-29T04:59:08.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[101970]: failed to encode map e401 with expected crc 2024-03-29T04:59:08.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[101970]: failed to encode map e401 with expected crc 2024-03-29T04:59:08.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[101970]: osdmap e401: 8 total, 8 up, 8 in 2024-03-29T04:59:08.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[101970]: failed to encode map e401 with expected crc 2024-03-29T04:59:08.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[101970]: failed to encode map e401 with expected crc 2024-03-29T04:59:08.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[101970]: failed to encode map e401 with expected crc 2024-03-29T04:59:08.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[101970]: failed to encode map e401 with expected crc 2024-03-29T04:59:08.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[101970]: failed to encode map e401 with expected crc 2024-03-29T04:59:08.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1338724449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:59:08.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[106466]: pgmap v254: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 341 op/s 2024-03-29T04:59:08.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[106466]: failed to encode map e401 with expected crc 2024-03-29T04:59:08.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[106466]: failed to encode map e401 with expected crc 2024-03-29T04:59:08.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[106466]: osdmap e401: 8 total, 8 up, 8 in 2024-03-29T04:59:08.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[106466]: failed to encode map e401 with expected crc 2024-03-29T04:59:08.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[106466]: failed to encode map e401 with expected crc 2024-03-29T04:59:08.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[106466]: failed to encode map e401 with expected crc 2024-03-29T04:59:08.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[106466]: failed to encode map e401 with expected crc 2024-03-29T04:59:08.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[106466]: failed to encode map e401 with expected crc 2024-03-29T04:59:08.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1338724449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:59:09.134 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:59:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:59:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:59:09.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:08 smithi187 ceph-mon[93536]: pgmap v254: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 341 op/s 2024-03-29T04:59:09.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:08 smithi187 ceph-mon[93536]: failed to encode map e401 with expected crc 2024-03-29T04:59:09.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:08 smithi187 ceph-mon[93536]: failed to encode map e401 with expected crc 2024-03-29T04:59:09.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:08 smithi187 ceph-mon[93536]: osdmap e401: 8 total, 8 up, 8 in 2024-03-29T04:59:09.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:08 smithi187 ceph-mon[93536]: failed to encode map e401 with expected crc 2024-03-29T04:59:09.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:08 smithi187 ceph-mon[93536]: failed to encode map e401 with expected crc 2024-03-29T04:59:09.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:08 smithi187 ceph-mon[93536]: failed to encode map e401 with expected crc 2024-03-29T04:59:09.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:08 smithi187 ceph-mon[93536]: failed to encode map e401 with expected crc 2024-03-29T04:59:09.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:08 smithi187 ceph-mon[93536]: failed to encode map e401 with expected crc 2024-03-29T04:59:09.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:08 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1338724449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:59:09.255 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.CloneFlatten (2191 ms) 2024-03-29T04:59:09.255 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.Stress 2024-03-29T04:59:09.276 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 39859669~5448517 2024-03-29T04:59:09.294 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:59:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:59:08] "GET /metrics HTTP/1.1" 200 36288 "" "Prometheus/2.43.0" 2024-03-29T04:59:09.297 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 21113428~7124882 2024-03-29T04:59:09.298 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 9936133~7647830 2024-03-29T04:59:09.320 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 72914333~380299 2024-03-29T04:59:09.320 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 33462157~2266976 2024-03-29T04:59:09.324 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T04:59:09.329 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 65116047~7389709 2024-03-29T04:59:09.329 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 7160222~445790 2024-03-29T04:59:09.332 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 53125317~3796569 2024-03-29T04:59:09.332 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap0 2024-03-29T04:59:09.670 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:59:09.762 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 115362858 2024-03-29T04:59:09.774 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 39738484~4276381 2024-03-29T04:59:09.774 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 12132196~2826635 2024-03-29T04:59:09.775 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 75305303~80490 2024-03-29T04:59:09.775 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 75283275~30397 2024-03-29T04:59:09.788 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 50245534~6186651 2024-03-29T04:59:09.789 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 47531932~2456994 2024-03-29T04:59:09.789 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 113502843~1519064 2024-03-29T04:59:09.789 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 37020086~4559142 2024-03-29T04:59:09.814 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap1 2024-03-29T04:59:10.000 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:09 smithi181 ceph-mon[106466]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.000 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:09 smithi181 ceph-mon[106466]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.000 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:09 smithi181 ceph-mon[106466]: osdmap e402: 8 total, 8 up, 8 in 2024-03-29T04:59:10.000 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:09 smithi181 ceph-mon[106466]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.000 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:09 smithi181 ceph-mon[106466]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.000 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:09 smithi181 ceph-mon[106466]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.000 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:09 smithi181 ceph-mon[106466]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.000 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:09 smithi181 ceph-mon[101970]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.001 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:09 smithi181 ceph-mon[101970]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.001 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:09 smithi181 ceph-mon[101970]: osdmap e402: 8 total, 8 up, 8 in 2024-03-29T04:59:10.001 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:09 smithi181 ceph-mon[101970]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.001 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:09 smithi181 ceph-mon[101970]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.001 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:09 smithi181 ceph-mon[101970]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.001 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:09 smithi181 ceph-mon[101970]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:09 smithi187 ceph-mon[93536]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:09 smithi187 ceph-mon[93536]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:09 smithi187 ceph-mon[93536]: osdmap e402: 8 total, 8 up, 8 in 2024-03-29T04:59:10.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:09 smithi187 ceph-mon[93536]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:09 smithi187 ceph-mon[93536]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:09 smithi187 ceph-mon[93536]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:09 smithi187 ceph-mon[93536]: failed to encode map e402 with expected crc 2024-03-29T04:59:10.779 INFO:tasks.workunit.client.0.smithi181.stdout:clone image240 -> image241 2024-03-29T04:59:10.835 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 198939583 2024-03-29T04:59:10.841 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 43087504~5438315 2024-03-29T04:59:10.869 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 93099514~6143301 2024-03-29T04:59:10.872 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 151582436~6380827 2024-03-29T04:59:10.890 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 24499450~3211030 2024-03-29T04:59:10.893 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 133044336~994152 2024-03-29T04:59:10.899 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 133066073~8199542 2024-03-29T04:59:10.901 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 89196816~6717579 2024-03-29T04:59:10.926 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 34092669~1743133 2024-03-29T04:59:10.936 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap2 2024-03-29T04:59:10.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:10 smithi181 ceph-mon[101970]: pgmap v257: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 343 op/s 2024-03-29T04:59:10.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:10 smithi181 ceph-mon[101970]: failed to encode map e403 with expected crc 2024-03-29T04:59:10.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:10 smithi181 ceph-mon[101970]: failed to encode map e403 with expected crc 2024-03-29T04:59:10.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:10 smithi181 ceph-mon[101970]: osdmap e403: 8 total, 8 up, 8 in 2024-03-29T04:59:10.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:10 smithi181 ceph-mon[101970]: failed to encode map e403 with expected crc 2024-03-29T04:59:10.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:10 smithi181 ceph-mon[101970]: failed to encode map e403 with expected crc 2024-03-29T04:59:10.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:10 smithi181 ceph-mon[101970]: failed to encode map e403 with expected crc 2024-03-29T04:59:10.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:10 smithi181 ceph-mon[106466]: pgmap v257: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 343 op/s 2024-03-29T04:59:10.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:10 smithi181 ceph-mon[106466]: failed to encode map e403 with expected crc 2024-03-29T04:59:10.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:10 smithi181 ceph-mon[106466]: failed to encode map e403 with expected crc 2024-03-29T04:59:10.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:10 smithi181 ceph-mon[106466]: osdmap e403: 8 total, 8 up, 8 in 2024-03-29T04:59:10.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:10 smithi181 ceph-mon[106466]: failed to encode map e403 with expected crc 2024-03-29T04:59:10.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:10 smithi181 ceph-mon[106466]: failed to encode map e403 with expected crc 2024-03-29T04:59:10.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:10 smithi181 ceph-mon[106466]: failed to encode map e403 with expected crc 2024-03-29T04:59:11.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:10 smithi187 ceph-mon[93536]: pgmap v257: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 343 op/s 2024-03-29T04:59:11.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:10 smithi187 ceph-mon[93536]: failed to encode map e403 with expected crc 2024-03-29T04:59:11.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:10 smithi187 ceph-mon[93536]: failed to encode map e403 with expected crc 2024-03-29T04:59:11.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:10 smithi187 ceph-mon[93536]: osdmap e403: 8 total, 8 up, 8 in 2024-03-29T04:59:11.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:10 smithi187 ceph-mon[93536]: failed to encode map e403 with expected crc 2024-03-29T04:59:11.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:10 smithi187 ceph-mon[93536]: failed to encode map e403 with expected crc 2024-03-29T04:59:11.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:10 smithi187 ceph-mon[93536]: failed to encode map e403 with expected crc 2024-03-29T04:59:11.822 INFO:tasks.workunit.client.0.smithi181.stdout:clone image241 -> image242 2024-03-29T04:59:11.832 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[101970]: failed to encode map e404 with expected crc 2024-03-29T04:59:11.832 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[101970]: failed to encode map e404 with expected crc 2024-03-29T04:59:11.832 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[101970]: osdmap e404: 8 total, 8 up, 8 in 2024-03-29T04:59:11.832 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[101970]: failed to encode map e404 with expected crc 2024-03-29T04:59:11.833 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[101970]: failed to encode map e404 with expected crc 2024-03-29T04:59:11.833 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[101970]: failed to encode map e404 with expected crc 2024-03-29T04:59:11.833 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:11.833 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:11.833 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:59:11.833 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[106466]: failed to encode map e404 with expected crc 2024-03-29T04:59:11.833 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[106466]: failed to encode map e404 with expected crc 2024-03-29T04:59:11.833 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[106466]: osdmap e404: 8 total, 8 up, 8 in 2024-03-29T04:59:11.833 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[106466]: failed to encode map e404 with expected crc 2024-03-29T04:59:11.833 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[106466]: failed to encode map e404 with expected crc 2024-03-29T04:59:11.834 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[106466]: failed to encode map e404 with expected crc 2024-03-29T04:59:11.834 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:11.834 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:11.834 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:59:11.881 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 104778367 2024-03-29T04:59:11.890 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 15034639~1939947 2024-03-29T04:59:11.927 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 5823135~2640779 2024-03-29T04:59:11.953 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 26002926~7561499 2024-03-29T04:59:11.979 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 100007803~4770564 2024-03-29T04:59:11.982 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 17224020~1626512 2024-03-29T04:59:11.989 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 45690056~1450093 2024-03-29T04:59:12.015 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 76932601~1453116 2024-03-29T04:59:12.024 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 27418683~5616780 2024-03-29T04:59:12.026 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap3 2024-03-29T04:59:12.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:11 smithi187 ceph-mon[93536]: failed to encode map e404 with expected crc 2024-03-29T04:59:12.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:11 smithi187 ceph-mon[93536]: failed to encode map e404 with expected crc 2024-03-29T04:59:12.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:11 smithi187 ceph-mon[93536]: osdmap e404: 8 total, 8 up, 8 in 2024-03-29T04:59:12.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:11 smithi187 ceph-mon[93536]: failed to encode map e404 with expected crc 2024-03-29T04:59:12.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:11 smithi187 ceph-mon[93536]: failed to encode map e404 with expected crc 2024-03-29T04:59:12.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:11 smithi187 ceph-mon[93536]: failed to encode map e404 with expected crc 2024-03-29T04:59:12.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:12.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:12.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:59:12.326 INFO:teuthology.orchestra.run.smithi181.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T04:59:12.326 INFO:teuthology.orchestra.run.smithi181.stdout:alertmanager.a smithi181 *:9093,9094 running (4m) 1s ago 8m 25.9M - 0.25.0 c8568f914cd2 69e0ad67b57f 2024-03-29T04:59:12.326 INFO:teuthology.orchestra.run.smithi181.stdout:grafana.a smithi187 *:3000 error 65s ago 8m - - 2024-03-29T04:59:12.326 INFO:teuthology.orchestra.run.smithi181.stdout:mds.foo.smithi181.pflrwm smithi181 running (7m) 1s ago 7m 16.9M - 17.2.7-789-g4213e17c cf22eb8683be 0f200e33dce6 2024-03-29T04:59:12.326 INFO:teuthology.orchestra.run.smithi181.stdout:mds.foo.smithi187.vcsnqx smithi187 running (7m) 65s ago 7m 16.7M - 17.2.7-789-g4213e17c cf22eb8683be 245621ed14ed 2024-03-29T04:59:12.327 INFO:teuthology.orchestra.run.smithi181.stdout:mgr.x smithi187 *:8443,9283,8765 running (3m) 65s ago 12m 449M - 19.0.0-1959-g8da23624 54112f57da44 19bc315720bd 2024-03-29T04:59:12.327 INFO:teuthology.orchestra.run.smithi181.stdout:mgr.y smithi181 *:8443,9283,8765 running (3m) 1s ago 14m 504M - 19.0.0-1959-g8da23624 54112f57da44 b07d84f4244a 2024-03-29T04:59:12.327 INFO:teuthology.orchestra.run.smithi181.stdout:mon.a smithi181 running (3m) 1s ago 14m 50.5M 2048M 19.0.0-1959-g8da23624 54112f57da44 4e5d3325d991 2024-03-29T04:59:12.327 INFO:teuthology.orchestra.run.smithi181.stdout:mon.b smithi187 running (72s) 65s ago 12m 28.4M 2048M 19.0.0-1959-g8da23624 54112f57da44 4de0ae3c44c3 2024-03-29T04:59:12.327 INFO:teuthology.orchestra.run.smithi181.stdout:mon.c smithi181 running (94s) 1s ago 12m 42.3M 2048M 19.0.0-1959-g8da23624 54112f57da44 10f02d533230 2024-03-29T04:59:12.327 INFO:teuthology.orchestra.run.smithi181.stdout:node-exporter.a smithi181 *:9100 running (4m) 1s ago 7m 16.4M - 1.5.0 0da6a335fe13 4029d029600b 2024-03-29T04:59:12.327 INFO:teuthology.orchestra.run.smithi181.stdout:node-exporter.b smithi187 *:9100 running (4m) 65s ago 7m 15.7M - 1.5.0 0da6a335fe13 3894f6e97d55 2024-03-29T04:59:12.327 INFO:teuthology.orchestra.run.smithi181.stdout:osd.0 smithi181 running (12m) 1s ago 12m 126M 2480M 17.2.7-789-g4213e17c cf22eb8683be ae426d29a1a8 2024-03-29T04:59:12.327 INFO:teuthology.orchestra.run.smithi181.stdout:osd.1 smithi181 running (11m) 1s ago 11m 113M 2480M 17.2.7-789-g4213e17c cf22eb8683be 7ed1966c65cf 2024-03-29T04:59:12.327 INFO:teuthology.orchestra.run.smithi181.stdout:osd.2 smithi181 running (11m) 1s ago 11m 122M 2480M 17.2.7-789-g4213e17c cf22eb8683be 9c61fb0d684c 2024-03-29T04:59:12.327 INFO:teuthology.orchestra.run.smithi181.stdout:osd.3 smithi181 running (10m) 1s ago 10m 118M 2480M 17.2.7-789-g4213e17c cf22eb8683be 6b0bf471cfcb 2024-03-29T04:59:12.327 INFO:teuthology.orchestra.run.smithi181.stdout:osd.4 smithi187 running (10m) 65s ago 10m 137M 2480M 17.2.7-789-g4213e17c cf22eb8683be 42698769ef84 2024-03-29T04:59:12.327 INFO:teuthology.orchestra.run.smithi181.stdout:osd.5 smithi187 running (10m) 65s ago 10m 113M 2480M 17.2.7-789-g4213e17c cf22eb8683be 9599ff85a9e7 2024-03-29T04:59:12.327 INFO:teuthology.orchestra.run.smithi181.stdout:osd.6 smithi187 running (9m) 65s ago 9m 139M 2480M 17.2.7-789-g4213e17c cf22eb8683be c3287470c1fa 2024-03-29T04:59:12.328 INFO:teuthology.orchestra.run.smithi181.stdout:osd.7 smithi187 running (9m) 65s ago 9m 114M 2480M 17.2.7-789-g4213e17c cf22eb8683be 89fb8ccf8705 2024-03-29T04:59:12.328 INFO:teuthology.orchestra.run.smithi181.stdout:prometheus.a smithi187 *:9095 running (3m) 65s ago 8m 57.3M - 2.43.0 a07b618ecd1d 882b8922d74b 2024-03-29T04:59:12.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: pgmap v260: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:59:12.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:59:12.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:59:12.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: pgmap v261: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:59:12.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: pgmap v262: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:59:12.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:12.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: failed to encode map e405 with expected crc 2024-03-29T04:59:12.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: failed to encode map e405 with expected crc 2024-03-29T04:59:12.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: osdmap e405: 8 total, 8 up, 8 in 2024-03-29T04:59:12.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: failed to encode map e405 with expected crc 2024-03-29T04:59:12.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: failed to encode map e405 with expected crc 2024-03-29T04:59:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:59:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:59:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:59:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: Upgrade: Setting container_image for all mon 2024-03-29T04:59:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-03-29T04:59:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-03-29T04:59:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-03-29T04:59:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-03-29T04:59:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-03-29T04:59:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-03-29T04:59:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:59:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: Upgrade: Setting container_image for all crash 2024-03-29T04:59:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:59:12.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: pgmap v260: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:59:12.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:59:12.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:59:12.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: pgmap v261: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:59:12.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: pgmap v262: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:59:12.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:12.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: failed to encode map e405 with expected crc 2024-03-29T04:59:12.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: failed to encode map e405 with expected crc 2024-03-29T04:59:12.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: osdmap e405: 8 total, 8 up, 8 in 2024-03-29T04:59:12.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: failed to encode map e405 with expected crc 2024-03-29T04:59:12.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: failed to encode map e405 with expected crc 2024-03-29T04:59:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:59:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:59:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:59:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: Upgrade: Setting container_image for all mon 2024-03-29T04:59:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-03-29T04:59:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-03-29T04:59:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-03-29T04:59:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-03-29T04:59:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-03-29T04:59:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-03-29T04:59:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:59:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: Upgrade: Setting container_image for all crash 2024-03-29T04:59:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:59:12.970 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 160890836 2024-03-29T04:59:13.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade pause' 2024-03-29T04:59:13.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: pgmap v260: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:59:13.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:59:13.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:59:13.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: pgmap v261: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:59:13.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: pgmap v262: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:59:13.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:13.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: failed to encode map e405 with expected crc 2024-03-29T04:59:13.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: failed to encode map e405 with expected crc 2024-03-29T04:59:13.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: osdmap e405: 8 total, 8 up, 8 in 2024-03-29T04:59:13.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: failed to encode map e405 with expected crc 2024-03-29T04:59:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: failed to encode map e405 with expected crc 2024-03-29T04:59:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:59:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:59:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:59:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: Upgrade: Setting container_image for all mon 2024-03-29T04:59:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-03-29T04:59:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-03-29T04:59:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-03-29T04:59:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-03-29T04:59:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-03-29T04:59:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-03-29T04:59:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:59:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: Upgrade: Setting container_image for all crash 2024-03-29T04:59:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:59:13.763 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:59:14.176 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:13 smithi181 ceph-mon[106466]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:59:14.176 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:13 smithi181 ceph-mon[106466]: Upgrade: osd.0 is safe to restart 2024-03-29T04:59:14.176 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:13 smithi181 ceph-mon[106466]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:59:14.177 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:13 smithi181 ceph-mon[106466]: failed to encode map e406 with expected crc 2024-03-29T04:59:14.177 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:13 smithi181 ceph-mon[106466]: failed to encode map e406 with expected crc 2024-03-29T04:59:14.177 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:13 smithi181 ceph-mon[106466]: osdmap e406: 8 total, 8 up, 8 in 2024-03-29T04:59:14.177 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:13 smithi181 ceph-mon[106466]: failed to encode map e406 with expected crc 2024-03-29T04:59:14.177 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:13 smithi181 ceph-mon[106466]: failed to encode map e406 with expected crc 2024-03-29T04:59:14.177 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:13 smithi181 ceph-mon[101970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:59:14.177 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:13 smithi181 ceph-mon[101970]: Upgrade: osd.0 is safe to restart 2024-03-29T04:59:14.177 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:13 smithi181 ceph-mon[101970]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:59:14.177 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:13 smithi181 ceph-mon[101970]: failed to encode map e406 with expected crc 2024-03-29T04:59:14.177 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:13 smithi181 ceph-mon[101970]: failed to encode map e406 with expected crc 2024-03-29T04:59:14.177 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:13 smithi181 ceph-mon[101970]: osdmap e406: 8 total, 8 up, 8 in 2024-03-29T04:59:14.177 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:13 smithi181 ceph-mon[101970]: failed to encode map e406 with expected crc 2024-03-29T04:59:14.178 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:13 smithi181 ceph-mon[101970]: failed to encode map e406 with expected crc 2024-03-29T04:59:14.365 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.Stress (5111 ms) 2024-03-29T04:59:14.365 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.NoSnaps_LargerDstObjSize 2024-03-29T04:59:14.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:13 smithi187 ceph-mon[93536]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:59:14.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:13 smithi187 ceph-mon[93536]: Upgrade: osd.0 is safe to restart 2024-03-29T04:59:14.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:13 smithi187 ceph-mon[93536]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:59:14.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:13 smithi187 ceph-mon[93536]: failed to encode map e406 with expected crc 2024-03-29T04:59:14.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:13 smithi187 ceph-mon[93536]: failed to encode map e406 with expected crc 2024-03-29T04:59:14.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:13 smithi187 ceph-mon[93536]: osdmap e406: 8 total, 8 up, 8 in 2024-03-29T04:59:14.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:13 smithi187 ceph-mon[93536]: failed to encode map e406 with expected crc 2024-03-29T04:59:14.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:13 smithi187 ceph-mon[93536]: failed to encode map e406 with expected crc 2024-03-29T04:59:14.648 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.NoSnaps_LargerDstObjSize (282 ms) 2024-03-29T04:59:14.648 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.Snaps_LargerDstObjSize 2024-03-29T04:59:15.048 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[106466]: pgmap v265: 105 pgs: 105 active+clean; 322 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s rd, 55 MiB/s wr, 692 op/s 2024-03-29T04:59:15.048 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[106466]: failed to encode map e407 with expected crc 2024-03-29T04:59:15.048 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[106466]: failed to encode map e407 with expected crc 2024-03-29T04:59:15.048 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[106466]: osdmap e407: 8 total, 8 up, 8 in 2024-03-29T04:59:15.048 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[106466]: failed to encode map e407 with expected crc 2024-03-29T04:59:15.048 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[106466]: failed to encode map e407 with expected crc 2024-03-29T04:59:15.049 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[106466]: failed to encode map e407 with expected crc 2024-03-29T04:59:15.049 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[106466]: Upgrade: Updating osd.0 2024-03-29T04:59:15.049 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:15.049 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:59:15.049 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:59:15.049 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[106466]: Deploying daemon osd.0 on smithi181 2024-03-29T04:59:15.049 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[101970]: pgmap v265: 105 pgs: 105 active+clean; 322 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s rd, 55 MiB/s wr, 692 op/s 2024-03-29T04:59:15.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[101970]: failed to encode map e407 with expected crc 2024-03-29T04:59:15.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[101970]: failed to encode map e407 with expected crc 2024-03-29T04:59:15.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[101970]: osdmap e407: 8 total, 8 up, 8 in 2024-03-29T04:59:15.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[101970]: failed to encode map e407 with expected crc 2024-03-29T04:59:15.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[101970]: failed to encode map e407 with expected crc 2024-03-29T04:59:15.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[101970]: failed to encode map e407 with expected crc 2024-03-29T04:59:15.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[101970]: Upgrade: Updating osd.0 2024-03-29T04:59:15.050 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:15.051 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:59:15.051 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:59:15.051 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:14 smithi181 ceph-mon[101970]: Deploying daemon osd.0 on smithi181 2024-03-29T04:59:15.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:14 smithi187 ceph-mon[93536]: pgmap v265: 105 pgs: 105 active+clean; 322 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 49 MiB/s rd, 55 MiB/s wr, 692 op/s 2024-03-29T04:59:15.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:14 smithi187 ceph-mon[93536]: failed to encode map e407 with expected crc 2024-03-29T04:59:15.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:14 smithi187 ceph-mon[93536]: failed to encode map e407 with expected crc 2024-03-29T04:59:15.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:14 smithi187 ceph-mon[93536]: osdmap e407: 8 total, 8 up, 8 in 2024-03-29T04:59:15.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:14 smithi187 ceph-mon[93536]: failed to encode map e407 with expected crc 2024-03-29T04:59:15.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:14 smithi187 ceph-mon[93536]: failed to encode map e407 with expected crc 2024-03-29T04:59:15.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:14 smithi187 ceph-mon[93536]: failed to encode map e407 with expected crc 2024-03-29T04:59:15.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:14 smithi187 ceph-mon[93536]: Upgrade: Updating osd.0 2024-03-29T04:59:15.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:14 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:15.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:14 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:59:15.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:14 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:59:15.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:14 smithi187 ceph-mon[93536]: Deploying daemon osd.0 on smithi181 2024-03-29T04:59:16.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:16 smithi181 ceph-mon[106466]: failed to encode map e408 with expected crc 2024-03-29T04:59:16.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:16 smithi181 ceph-mon[106466]: failed to encode map e408 with expected crc 2024-03-29T04:59:16.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:16 smithi181 ceph-mon[106466]: osdmap e408: 8 total, 8 up, 8 in 2024-03-29T04:59:16.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:16 smithi181 ceph-mon[106466]: failed to encode map e408 with expected crc 2024-03-29T04:59:16.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:16 smithi181 ceph-mon[106466]: failed to encode map e408 with expected crc 2024-03-29T04:59:16.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:16 smithi181 ceph-mon[106466]: pgmap v268: 105 pgs: 105 active+clean; 322 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s rd, 42 MiB/s wr, 521 op/s 2024-03-29T04:59:16.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:16 smithi181 ceph-mon[101970]: failed to encode map e408 with expected crc 2024-03-29T04:59:16.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:16 smithi181 ceph-mon[101970]: failed to encode map e408 with expected crc 2024-03-29T04:59:16.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:16 smithi181 ceph-mon[101970]: osdmap e408: 8 total, 8 up, 8 in 2024-03-29T04:59:16.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:16 smithi181 ceph-mon[101970]: failed to encode map e408 with expected crc 2024-03-29T04:59:16.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:16 smithi181 ceph-mon[101970]: failed to encode map e408 with expected crc 2024-03-29T04:59:16.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:16 smithi181 ceph-mon[101970]: pgmap v268: 105 pgs: 105 active+clean; 322 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s rd, 42 MiB/s wr, 521 op/s 2024-03-29T04:59:16.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:16 smithi187 ceph-mon[93536]: failed to encode map e408 with expected crc 2024-03-29T04:59:16.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:16 smithi187 ceph-mon[93536]: failed to encode map e408 with expected crc 2024-03-29T04:59:16.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:16 smithi187 ceph-mon[93536]: osdmap e408: 8 total, 8 up, 8 in 2024-03-29T04:59:16.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:16 smithi187 ceph-mon[93536]: failed to encode map e408 with expected crc 2024-03-29T04:59:16.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:16 smithi187 ceph-mon[93536]: failed to encode map e408 with expected crc 2024-03-29T04:59:16.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:16 smithi187 ceph-mon[93536]: pgmap v268: 105 pgs: 105 active+clean; 322 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s rd, 42 MiB/s wr, 521 op/s 2024-03-29T04:59:17.305 INFO:teuthology.orchestra.run.smithi181.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:59:17.317 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:17 smithi181 ceph-mon[101970]: failed to encode map e408 with expected crc 2024-03-29T04:59:17.317 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:17 smithi181 ceph-mon[101970]: failed to encode map e409 with expected crc 2024-03-29T04:59:17.317 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:17 smithi181 ceph-mon[101970]: failed to encode map e409 with expected crc 2024-03-29T04:59:17.318 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:17 smithi181 ceph-mon[101970]: osdmap e409: 8 total, 8 up, 8 in 2024-03-29T04:59:17.318 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:17 smithi181 ceph-mon[101970]: failed to encode map e409 with expected crc 2024-03-29T04:59:17.318 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:17 smithi181 ceph-mon[101970]: failed to encode map e409 with expected crc 2024-03-29T04:59:17.318 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:17 smithi181 ceph-mon[106466]: failed to encode map e408 with expected crc 2024-03-29T04:59:17.318 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:17 smithi181 ceph-mon[106466]: failed to encode map e409 with expected crc 2024-03-29T04:59:17.318 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:17 smithi181 ceph-mon[106466]: failed to encode map e409 with expected crc 2024-03-29T04:59:17.318 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:17 smithi181 ceph-mon[106466]: osdmap e409: 8 total, 8 up, 8 in 2024-03-29T04:59:17.319 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:17 smithi181 ceph-mon[106466]: failed to encode map e409 with expected crc 2024-03-29T04:59:17.319 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:17 smithi181 ceph-mon[106466]: failed to encode map e409 with expected crc 2024-03-29T04:59:17.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:17 smithi187 ceph-mon[93536]: failed to encode map e408 with expected crc 2024-03-29T04:59:17.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:17 smithi187 ceph-mon[93536]: failed to encode map e409 with expected crc 2024-03-29T04:59:17.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:17 smithi187 ceph-mon[93536]: failed to encode map e409 with expected crc 2024-03-29T04:59:17.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:17 smithi187 ceph-mon[93536]: osdmap e409: 8 total, 8 up, 8 in 2024-03-29T04:59:17.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:17 smithi187 ceph-mon[93536]: failed to encode map e409 with expected crc 2024-03-29T04:59:17.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:17 smithi187 ceph-mon[93536]: failed to encode map e409 with expected crc 2024-03-29T04:59:17.954 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'sleep 60' 2024-03-29T04:59:18.311 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:59:18.576 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[106466]: failed to encode map e409 with expected crc 2024-03-29T04:59:18.576 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[106466]: failed to encode map e410 with expected crc 2024-03-29T04:59:18.576 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[106466]: failed to encode map e410 with expected crc 2024-03-29T04:59:18.577 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[106466]: osdmap e410: 8 total, 8 up, 8 in 2024-03-29T04:59:18.577 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[106466]: failed to encode map e410 with expected crc 2024-03-29T04:59:18.577 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[106466]: failed to encode map e410 with expected crc 2024-03-29T04:59:18.577 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[106466]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:59:18.577 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[106466]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:59:18.577 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:18.577 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[106466]: failed to encode map e410 with expected crc 2024-03-29T04:59:18.577 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[106466]: pgmap v271: 105 pgs: 105 active+clean; 399 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 51 MiB/s rd, 17 MiB/s wr, 330 op/s 2024-03-29T04:59:18.578 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[101970]: failed to encode map e409 with expected crc 2024-03-29T04:59:18.578 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[101970]: failed to encode map e410 with expected crc 2024-03-29T04:59:18.578 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[101970]: failed to encode map e410 with expected crc 2024-03-29T04:59:18.578 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[101970]: osdmap e410: 8 total, 8 up, 8 in 2024-03-29T04:59:18.578 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[101970]: failed to encode map e410 with expected crc 2024-03-29T04:59:18.578 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[101970]: failed to encode map e410 with expected crc 2024-03-29T04:59:18.578 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[101970]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:59:18.578 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[101970]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:59:18.579 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:18.579 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[101970]: failed to encode map e410 with expected crc 2024-03-29T04:59:18.579 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-mon[101970]: pgmap v271: 105 pgs: 105 active+clean; 399 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 51 MiB/s rd, 17 MiB/s wr, 330 op/s 2024-03-29T04:59:18.581 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.Snaps_LargerDstObjSize (3933 ms) 2024-03-29T04:59:18.581 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.Clone_LargerDstObjSize 2024-03-29T04:59:18.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:18 smithi187 ceph-mon[93536]: failed to encode map e409 with expected crc 2024-03-29T04:59:18.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:18 smithi187 ceph-mon[93536]: failed to encode map e410 with expected crc 2024-03-29T04:59:18.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:18 smithi187 ceph-mon[93536]: failed to encode map e410 with expected crc 2024-03-29T04:59:18.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:18 smithi187 ceph-mon[93536]: osdmap e410: 8 total, 8 up, 8 in 2024-03-29T04:59:18.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:18 smithi187 ceph-mon[93536]: failed to encode map e410 with expected crc 2024-03-29T04:59:18.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:18 smithi187 ceph-mon[93536]: failed to encode map e410 with expected crc 2024-03-29T04:59:18.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:18 smithi187 ceph-mon[93536]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:59:18.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:18 smithi187 ceph-mon[93536]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:59:18.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:18 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:18.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:18 smithi187 ceph-mon[93536]: failed to encode map e410 with expected crc 2024-03-29T04:59:18.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:18 smithi187 ceph-mon[93536]: pgmap v271: 105 pgs: 105 active+clean; 399 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 51 MiB/s rd, 17 MiB/s wr, 330 op/s 2024-03-29T04:59:19.121 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:59:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:59:18] "GET /metrics HTTP/1.1" 200 36294 "" "Prometheus/2.43.0" 2024-03-29T04:59:19.133 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:59:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:59:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:59:19.406 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:19 smithi181 ceph-mon[106466]: failed to encode map e411 with expected crc 2024-03-29T04:59:19.407 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:19 smithi181 ceph-mon[106466]: failed to encode map e411 with expected crc 2024-03-29T04:59:19.407 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:19 smithi181 ceph-mon[106466]: osdmap e411: 8 total, 8 up, 8 in 2024-03-29T04:59:19.407 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:19 smithi181 ceph-mon[106466]: failed to encode map e411 with expected crc 2024-03-29T04:59:19.407 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:19 smithi181 ceph-mon[106466]: failed to encode map e411 with expected crc 2024-03-29T04:59:19.407 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:19 smithi181 ceph-mon[101970]: failed to encode map e411 with expected crc 2024-03-29T04:59:19.407 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:19 smithi181 ceph-mon[101970]: failed to encode map e411 with expected crc 2024-03-29T04:59:19.407 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:19 smithi181 ceph-mon[101970]: osdmap e411: 8 total, 8 up, 8 in 2024-03-29T04:59:19.407 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:19 smithi181 ceph-mon[101970]: failed to encode map e411 with expected crc 2024-03-29T04:59:19.407 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:19 smithi181 ceph-mon[101970]: failed to encode map e411 with expected crc 2024-03-29T04:59:19.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:19 smithi187 ceph-mon[93536]: failed to encode map e411 with expected crc 2024-03-29T04:59:19.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:19 smithi187 ceph-mon[93536]: failed to encode map e411 with expected crc 2024-03-29T04:59:19.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:19 smithi187 ceph-mon[93536]: osdmap e411: 8 total, 8 up, 8 in 2024-03-29T04:59:19.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:19 smithi187 ceph-mon[93536]: failed to encode map e411 with expected crc 2024-03-29T04:59:19.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:19 smithi187 ceph-mon[93536]: failed to encode map e411 with expected crc 2024-03-29T04:59:19.703 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:19 smithi181 systemd[1]: Stopping Ceph osd.0 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:59:20.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:20 smithi181 ceph-mon[101970]: failed to encode map e412 with expected crc 2024-03-29T04:59:20.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:20 smithi181 ceph-mon[101970]: failed to encode map e412 with expected crc 2024-03-29T04:59:20.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:20 smithi181 ceph-mon[101970]: osdmap e412: 8 total, 8 up, 8 in 2024-03-29T04:59:20.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:20 smithi181 ceph-mon[101970]: failed to encode map e412 with expected crc 2024-03-29T04:59:20.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:20 smithi181 ceph-mon[101970]: failed to encode map e412 with expected crc 2024-03-29T04:59:20.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:20 smithi181 ceph-mon[101970]: pgmap v274: 105 pgs: 105 active+clean; 399 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 51 MiB/s rd, 17 MiB/s wr, 330 op/s 2024-03-29T04:59:20.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:20 smithi181 ceph-mon[106466]: failed to encode map e412 with expected crc 2024-03-29T04:59:20.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:20 smithi181 ceph-mon[106466]: failed to encode map e412 with expected crc 2024-03-29T04:59:20.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:20 smithi181 ceph-mon[106466]: osdmap e412: 8 total, 8 up, 8 in 2024-03-29T04:59:20.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:20 smithi181 ceph-mon[106466]: failed to encode map e412 with expected crc 2024-03-29T04:59:20.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:20 smithi181 ceph-mon[106466]: failed to encode map e412 with expected crc 2024-03-29T04:59:20.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:20 smithi181 ceph-mon[106466]: pgmap v274: 105 pgs: 105 active+clean; 399 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 51 MiB/s rd, 17 MiB/s wr, 330 op/s 2024-03-29T04:59:20.455 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0[45500]: 2024-03-29T04:59:20.326+0000 7f0889323700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:59:20.455 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0[45500]: 2024-03-29T04:59:20.326+0000 7f0889323700 -1 osd.0 412 *** Got signal Terminated *** 2024-03-29T04:59:20.455 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0[45500]: 2024-03-29T04:59:20.326+0000 7f0889323700 -1 osd.0 412 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:59:20.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:20 smithi187 ceph-mon[93536]: failed to encode map e412 with expected crc 2024-03-29T04:59:20.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:20 smithi187 ceph-mon[93536]: failed to encode map e412 with expected crc 2024-03-29T04:59:20.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:20 smithi187 ceph-mon[93536]: osdmap e412: 8 total, 8 up, 8 in 2024-03-29T04:59:20.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:20 smithi187 ceph-mon[93536]: failed to encode map e412 with expected crc 2024-03-29T04:59:20.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:20 smithi187 ceph-mon[93536]: failed to encode map e412 with expected crc 2024-03-29T04:59:20.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:20 smithi187 ceph-mon[93536]: pgmap v274: 105 pgs: 105 active+clean; 399 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 51 MiB/s rd, 17 MiB/s wr, 330 op/s 2024-03-29T04:59:20.669 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.Clone_LargerDstObjSize (2088 ms) 2024-03-29T04:59:20.669 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.CloneFlatten_LargerDstObjSize 2024-03-29T04:59:21.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:21 smithi187 ceph-mon[93536]: failed to encode map e413 with expected crc 2024-03-29T04:59:21.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:21 smithi187 ceph-mon[93536]: failed to encode map e413 with expected crc 2024-03-29T04:59:21.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:21 smithi187 ceph-mon[93536]: osdmap e413: 8 total, 8 up, 8 in 2024-03-29T04:59:21.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:21 smithi187 ceph-mon[93536]: failed to encode map e413 with expected crc 2024-03-29T04:59:21.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:21 smithi187 ceph-mon[93536]: osd.0 marked itself down and dead 2024-03-29T04:59:21.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:21 smithi181 ceph-mon[101970]: failed to encode map e413 with expected crc 2024-03-29T04:59:21.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:21 smithi181 ceph-mon[101970]: failed to encode map e413 with expected crc 2024-03-29T04:59:21.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:21 smithi181 ceph-mon[101970]: osdmap e413: 8 total, 8 up, 8 in 2024-03-29T04:59:21.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:21 smithi181 ceph-mon[101970]: failed to encode map e413 with expected crc 2024-03-29T04:59:21.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:21 smithi181 ceph-mon[101970]: osd.0 marked itself down and dead 2024-03-29T04:59:21.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:21 smithi181 ceph-mon[106466]: failed to encode map e413 with expected crc 2024-03-29T04:59:21.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:21 smithi181 ceph-mon[106466]: failed to encode map e413 with expected crc 2024-03-29T04:59:21.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:21 smithi181 ceph-mon[106466]: osdmap e413: 8 total, 8 up, 8 in 2024-03-29T04:59:21.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:21 smithi181 ceph-mon[106466]: failed to encode map e413 with expected crc 2024-03-29T04:59:21.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:21 smithi181 ceph-mon[106466]: osd.0 marked itself down and dead 2024-03-29T04:59:22.162 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:21 smithi181 podman[114541]: 2024-03-29 04:59:21.835092834 +0000 UTC m=+1.634340015 container died ae426d29a1a8fe2089caaa03ad5c95c1c9066bb218ce53d0d59cdfd6f06f64a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD) 2024-03-29T04:59:22.453 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:22 smithi181 podman[114541]: 2024-03-29 04:59:22.253731802 +0000 UTC m=+2.052978989 container cleanup ae426d29a1a8fe2089caaa03ad5c95c1c9066bb218ce53d0d59cdfd6f06f64a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-03-29T04:59:22.453 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:22 smithi181 bash[114541]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0 2024-03-29T04:59:22.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:22 smithi181 ceph-mon[106466]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:59:22.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:22 smithi181 ceph-mon[106466]: failed to encode map e414 with expected crc 2024-03-29T04:59:22.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:22 smithi181 ceph-mon[106466]: failed to encode map e414 with expected crc 2024-03-29T04:59:22.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:22 smithi181 ceph-mon[106466]: osdmap e414: 8 total, 7 up, 8 in 2024-03-29T04:59:22.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:22 smithi181 ceph-mon[106466]: failed to encode map e414 with expected crc 2024-03-29T04:59:22.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:22 smithi181 ceph-mon[106466]: pgmap v277: 105 pgs: 13 stale+active+clean, 92 active+clean; 399 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:59:22.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:22 smithi181 ceph-mon[101970]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:59:22.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:22 smithi181 ceph-mon[101970]: failed to encode map e414 with expected crc 2024-03-29T04:59:22.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:22 smithi181 ceph-mon[101970]: failed to encode map e414 with expected crc 2024-03-29T04:59:22.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:22 smithi181 ceph-mon[101970]: osdmap e414: 8 total, 7 up, 8 in 2024-03-29T04:59:22.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:22 smithi181 ceph-mon[101970]: failed to encode map e414 with expected crc 2024-03-29T04:59:22.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:22 smithi181 ceph-mon[101970]: pgmap v277: 105 pgs: 13 stale+active+clean, 92 active+clean; 399 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:59:22.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:22 smithi187 ceph-mon[93536]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:59:22.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:22 smithi187 ceph-mon[93536]: failed to encode map e414 with expected crc 2024-03-29T04:59:22.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:22 smithi187 ceph-mon[93536]: failed to encode map e414 with expected crc 2024-03-29T04:59:22.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:22 smithi187 ceph-mon[93536]: osdmap e414: 8 total, 7 up, 8 in 2024-03-29T04:59:22.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:22 smithi187 ceph-mon[93536]: failed to encode map e414 with expected crc 2024-03-29T04:59:22.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:22 smithi187 ceph-mon[93536]: pgmap v277: 105 pgs: 13 stale+active+clean, 92 active+clean; 399 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-03-29T04:59:22.919 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:22 smithi181 podman[114603]: 2024-03-29 04:59:22.497188002 +0000 UTC m=+0.662973853 container remove ae426d29a1a8fe2089caaa03ad5c95c1c9066bb218ce53d0d59cdfd6f06f64a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=quincy-4213e17, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:59:23.172 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:22 smithi181 podman[114700]: 2024-03-29 04:59:22.815482286 +0000 UTC m=+0.017889244 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:59:23.173 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:22 smithi181 podman[114700]: 2024-03-29 04:59:22.931540593 +0000 UTC m=+0.133947538 container create c9bea0ddbfa0fec6699942b4080266ddccd6ca3705a6f1ac61d5bb9c7fd1ecb5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-03-29T04:59:23.173 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:23 smithi181 podman[114700]: 2024-03-29 04:59:23.159275896 +0000 UTC m=+0.361682841 container init c9bea0ddbfa0fec6699942b4080266ddccd6ca3705a6f1ac61d5bb9c7fd1ecb5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0) 2024-03-29T04:59:23.173 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:23 smithi181 podman[114700]: 2024-03-29 04:59:23.165154574 +0000 UTC m=+0.367561522 container start c9bea0ddbfa0fec6699942b4080266ddccd6ca3705a6f1ac61d5bb9c7fd1ecb5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362) 2024-03-29T04:59:23.453 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:23 smithi181 podman[114700]: 2024-03-29 04:59:23.267608607 +0000 UTC m=+0.470015561 container attach c9bea0ddbfa0fec6699942b4080266ddccd6ca3705a6f1ac61d5bb9c7fd1ecb5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:59:23.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[101970]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[101970]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[101970]: osdmap e415: 8 total, 7 up, 8 in 2024-03-29T04:59:23.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[101970]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[101970]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[101970]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[101970]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[101970]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[101970]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[101970]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[106466]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[106466]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[106466]: osdmap e415: 8 total, 7 up, 8 in 2024-03-29T04:59:23.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[106466]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[106466]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[106466]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[106466]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[106466]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[106466]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:23 smithi181 ceph-mon[106466]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:23 smithi187 ceph-mon[93536]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:23 smithi187 ceph-mon[93536]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:23 smithi187 ceph-mon[93536]: osdmap e415: 8 total, 7 up, 8 in 2024-03-29T04:59:23.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:23 smithi187 ceph-mon[93536]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:23 smithi187 ceph-mon[93536]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:23 smithi187 ceph-mon[93536]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:23 smithi187 ceph-mon[93536]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:23 smithi187 ceph-mon[93536]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:23 smithi187 ceph-mon[93536]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:23 smithi187 ceph-mon[93536]: failed to encode map e415 with expected crc 2024-03-29T04:59:23.862 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.CloneFlatten_LargerDstObjSize (3193 ms) 2024-03-29T04:59:23.862 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.Stress_LargerDstObjSize 2024-03-29T04:59:23.871 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:23 smithi181 podman[114700]: 2024-03-29 04:59:23.480144747 +0000 UTC m=+0.682551701 container died c9bea0ddbfa0fec6699942b4080266ddccd6ca3705a6f1ac61d5bb9c7fd1ecb5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240326, ceph=True, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:59:23.890 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 3727913~1027070 2024-03-29T04:59:23.896 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 30344202~1485124 2024-03-29T04:59:23.897 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 75133824~5471047 2024-03-29T04:59:23.916 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 17800242~1021568 2024-03-29T04:59:23.917 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 11350123~6206585 2024-03-29T04:59:23.935 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 458756~2761656 2024-03-29T04:59:23.937 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 76527624~7358456 2024-03-29T04:59:23.961 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 54072060~4618600 2024-03-29T04:59:23.961 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap0 2024-03-29T04:59:24.356 INFO:tasks.workunit.client.0.smithi181.stdout:clone image261 -> image262 2024-03-29T04:59:24.401 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 65836930~8143822 2024-03-29T04:59:24.432 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 68317194~5718591 2024-03-29T04:59:24.433 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 19723980~163195 2024-03-29T04:59:24.441 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 66699407~7100863 2024-03-29T04:59:24.443 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 58147465~6764027 2024-03-29T04:59:24.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:24 smithi181 ceph-mon[106466]: failed to encode map e416 with expected crc 2024-03-29T04:59:24.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:24 smithi181 ceph-mon[106466]: failed to encode map e416 with expected crc 2024-03-29T04:59:24.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:24 smithi181 ceph-mon[106466]: osdmap e416: 8 total, 7 up, 8 in 2024-03-29T04:59:24.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:24 smithi181 ceph-mon[106466]: failed to encode map e416 with expected crc 2024-03-29T04:59:24.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:24 smithi181 ceph-mon[106466]: failed to encode map e416 with expected crc 2024-03-29T04:59:24.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:24 smithi181 ceph-mon[106466]: pgmap v280: 105 pgs: 36 peering, 69 active+clean; 475 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 19 MiB/s wr, 315 op/s 2024-03-29T04:59:24.454 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:24 smithi181 podman[114746]: 2024-03-29 04:59:24.164180392 +0000 UTC m=+0.675673791 container remove c9bea0ddbfa0fec6699942b4080266ddccd6ca3705a6f1ac61d5bb9c7fd1ecb5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-deactivate, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, RELEASE=squid-8da2362, org.label-schema.license=GPLv2) 2024-03-29T04:59:24.454 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:24 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.0.service: Deactivated successfully. 2024-03-29T04:59:24.454 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:24 smithi181 systemd[1]: Stopped Ceph osd.0 for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:59:24.454 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:24 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.0.service: Consumed 22.592s CPU time. 2024-03-29T04:59:24.454 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:24 smithi181 systemd[1]: Starting Ceph osd.0 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T04:59:24.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:24 smithi181 ceph-mon[101970]: failed to encode map e416 with expected crc 2024-03-29T04:59:24.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:24 smithi181 ceph-mon[101970]: failed to encode map e416 with expected crc 2024-03-29T04:59:24.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:24 smithi181 ceph-mon[101970]: osdmap e416: 8 total, 7 up, 8 in 2024-03-29T04:59:24.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:24 smithi181 ceph-mon[101970]: failed to encode map e416 with expected crc 2024-03-29T04:59:24.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:24 smithi181 ceph-mon[101970]: failed to encode map e416 with expected crc 2024-03-29T04:59:24.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:24 smithi181 ceph-mon[101970]: pgmap v280: 105 pgs: 36 peering, 69 active+clean; 475 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 19 MiB/s wr, 315 op/s 2024-03-29T04:59:24.460 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 61907091~806679 2024-03-29T04:59:24.462 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 18610966~2583362 2024-03-29T04:59:24.470 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 76971332~6914748 2024-03-29T04:59:24.479 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap1 2024-03-29T04:59:24.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:24 smithi187 ceph-mon[93536]: failed to encode map e416 with expected crc 2024-03-29T04:59:24.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:24 smithi187 ceph-mon[93536]: failed to encode map e416 with expected crc 2024-03-29T04:59:24.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:24 smithi187 ceph-mon[93536]: osdmap e416: 8 total, 7 up, 8 in 2024-03-29T04:59:24.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:24 smithi187 ceph-mon[93536]: failed to encode map e416 with expected crc 2024-03-29T04:59:24.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:24 smithi187 ceph-mon[93536]: failed to encode map e416 with expected crc 2024-03-29T04:59:24.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:24 smithi187 ceph-mon[93536]: pgmap v280: 105 pgs: 36 peering, 69 active+clean; 475 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 19 MiB/s wr, 315 op/s 2024-03-29T04:59:24.881 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:24 smithi181 podman[114850]: 2024-03-29 04:59:24.525676998 +0000 UTC m=+0.017986630 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:59:24.881 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:24 smithi181 podman[114850]: 2024-03-29 04:59:24.749479368 +0000 UTC m=+0.241788975 container create edab02391498c5481cd98bf7e58caf4d4859d9d25a85bc69138e0f0118316063 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-activate, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:59:25.334 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:24 smithi181 podman[114850]: 2024-03-29 04:59:24.977378753 +0000 UTC m=+0.469688385 container init edab02391498c5481cd98bf7e58caf4d4859d9d25a85bc69138e0f0118316063 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-activate, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-03-29T04:59:25.334 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:24 smithi181 podman[114850]: 2024-03-29 04:59:24.983462965 +0000 UTC m=+0.475772574 container start edab02391498c5481cd98bf7e58caf4d4859d9d25a85bc69138e0f0118316063 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-activate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240326, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:59:25.334 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:25 smithi181 podman[114850]: 2024-03-29 04:59:25.085180298 +0000 UTC m=+0.577489909 container attach edab02391498c5481cd98bf7e58caf4d4859d9d25a85bc69138e0f0118316063 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-activate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, ceph=True, org.label-schema.build-date=20240326, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-03-29T04:59:25.351 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 154699782 2024-03-29T04:59:25.357 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 22008319~8033697 2024-03-29T04:59:25.359 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 133330384~1522815 2024-03-29T04:59:25.362 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 28337217~7781870 2024-03-29T04:59:25.390 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 102201995~1576453 2024-03-29T04:59:25.392 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 136808687~6600171 2024-03-29T04:59:25.393 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 90511509~6994390 2024-03-29T04:59:25.395 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 22710562~5072079 2024-03-29T04:59:25.395 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 76326070~1786659 2024-03-29T04:59:25.420 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap2 2024-03-29T04:59:25.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:25 smithi187 ceph-mon[93536]: Health check failed: Reduced data availability: 9 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-03-29T04:59:25.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:25 smithi187 ceph-mon[93536]: failed to encode map e417 with expected crc 2024-03-29T04:59:25.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:25 smithi187 ceph-mon[93536]: failed to encode map e417 with expected crc 2024-03-29T04:59:25.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:25 smithi187 ceph-mon[93536]: osdmap e417: 8 total, 7 up, 8 in 2024-03-29T04:59:25.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:25 smithi187 ceph-mon[93536]: failed to encode map e417 with expected crc 2024-03-29T04:59:25.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:25 smithi187 ceph-mon[93536]: failed to encode map e417 with expected crc 2024-03-29T04:59:25.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:25 smithi187 ceph-mon[93536]: failed to encode map e417 with expected crc 2024-03-29T04:59:25.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-mon[106466]: Health check failed: Reduced data availability: 9 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-03-29T04:59:25.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-mon[106466]: failed to encode map e417 with expected crc 2024-03-29T04:59:25.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-mon[106466]: failed to encode map e417 with expected crc 2024-03-29T04:59:25.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-mon[106466]: osdmap e417: 8 total, 7 up, 8 in 2024-03-29T04:59:25.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-mon[106466]: failed to encode map e417 with expected crc 2024-03-29T04:59:25.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-mon[106466]: failed to encode map e417 with expected crc 2024-03-29T04:59:25.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-mon[106466]: failed to encode map e417 with expected crc 2024-03-29T04:59:25.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-mon[101970]: Health check failed: Reduced data availability: 9 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-03-29T04:59:25.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-mon[101970]: failed to encode map e417 with expected crc 2024-03-29T04:59:25.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-mon[101970]: failed to encode map e417 with expected crc 2024-03-29T04:59:25.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-mon[101970]: osdmap e417: 8 total, 7 up, 8 in 2024-03-29T04:59:25.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-mon[101970]: failed to encode map e417 with expected crc 2024-03-29T04:59:25.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-mon[101970]: failed to encode map e417 with expected crc 2024-03-29T04:59:25.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-mon[101970]: failed to encode map e417 with expected crc 2024-03-29T04:59:25.705 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-activate[114865]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-29T04:59:25.705 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-activate[114865]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-29T04:59:25.705 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:25 smithi181 bash[114850]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-29T04:59:25.705 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:25 smithi181 bash[114850]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-29T04:59:25.705 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:25 smithi181 bash[114850]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-29T04:59:25.705 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:25 smithi181 bash[114850]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-29T04:59:25.705 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:25 smithi181 bash[114850]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-03-29T04:59:25.706 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:25 smithi181 bash[114850]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-29T04:59:25.706 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:25 smithi181 bash[114850]: --> ceph-volume raw activate successful for osd ID: 0 2024-03-29T04:59:25.706 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-activate[114865]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-29T04:59:25.706 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-activate[114865]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-29T04:59:25.706 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-activate[114865]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-03-29T04:59:25.706 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-activate[114865]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-29T04:59:25.706 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:25 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-activate[114865]: --> ceph-volume raw activate successful for osd ID: 0 2024-03-29T04:59:25.706 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:25 smithi181 podman[114850]: 2024-03-29 04:59:25.470974161 +0000 UTC m=+0.963283780 container died edab02391498c5481cd98bf7e58caf4d4859d9d25a85bc69138e0f0118316063 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-03-29T04:59:26.371 INFO:tasks.workunit.client.0.smithi181.stdout:clone image262 -> image263 2024-03-29T04:59:26.439 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 184090229 2024-03-29T04:59:26.447 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 178614035~5476194 2024-03-29T04:59:26.471 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 16967217~4419874 2024-03-29T04:59:26.490 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 134209797~6295677 2024-03-29T04:59:26.514 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 75144711~8116624 2024-03-29T04:59:26.519 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 101769714~2683114 2024-03-29T04:59:26.527 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 134053660~430397 2024-03-29T04:59:26.529 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 16134954~6385233 2024-03-29T04:59:26.566 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 63914956~7910767 2024-03-29T04:59:26.577 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap3 2024-03-29T04:59:26.615 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:26 smithi181 podman[115012]: 2024-03-29 04:59:26.356455735 +0000 UTC m=+0.880707946 container remove edab02391498c5481cd98bf7e58caf4d4859d9d25a85bc69138e0f0118316063 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-03-29T04:59:26.615 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:26 smithi181 podman[115081]: 2024-03-29 04:59:26.483392816 +0000 UTC m=+0.020912169 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:59:26.615 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:26 smithi181 podman[115081]: 2024-03-29 04:59:26.607802281 +0000 UTC m=+0.145321633 container create 03d36b79ef26d0e5c3783209243761f73aed0cc0e7999558db8183342e0adb87 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-03-29T04:59:26.615 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:26 smithi181 ceph-mon[101970]: failed to encode map e418 with expected crc 2024-03-29T04:59:26.615 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:26 smithi181 ceph-mon[101970]: failed to encode map e418 with expected crc 2024-03-29T04:59:26.615 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:26 smithi181 ceph-mon[101970]: osdmap e418: 8 total, 7 up, 8 in 2024-03-29T04:59:26.615 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:26 smithi181 ceph-mon[101970]: failed to encode map e418 with expected crc 2024-03-29T04:59:26.615 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:26 smithi181 ceph-mon[101970]: pgmap v283: 105 pgs: 36 peering, 69 active+clean; 475 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 19 MiB/s wr, 315 op/s 2024-03-29T04:59:26.616 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:26.616 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:59:26.616 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:26 smithi181 ceph-mon[106466]: failed to encode map e418 with expected crc 2024-03-29T04:59:26.616 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:26 smithi181 ceph-mon[106466]: failed to encode map e418 with expected crc 2024-03-29T04:59:26.616 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:26 smithi181 ceph-mon[106466]: osdmap e418: 8 total, 7 up, 8 in 2024-03-29T04:59:26.616 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:26 smithi181 ceph-mon[106466]: failed to encode map e418 with expected crc 2024-03-29T04:59:26.616 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:26 smithi181 ceph-mon[106466]: pgmap v283: 105 pgs: 36 peering, 69 active+clean; 475 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 19 MiB/s wr, 315 op/s 2024-03-29T04:59:26.616 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:26.616 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:59:26.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:26 smithi187 ceph-mon[93536]: failed to encode map e418 with expected crc 2024-03-29T04:59:26.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:26 smithi187 ceph-mon[93536]: failed to encode map e418 with expected crc 2024-03-29T04:59:26.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:26 smithi187 ceph-mon[93536]: osdmap e418: 8 total, 7 up, 8 in 2024-03-29T04:59:26.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:26 smithi187 ceph-mon[93536]: failed to encode map e418 with expected crc 2024-03-29T04:59:26.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:26 smithi187 ceph-mon[93536]: pgmap v283: 105 pgs: 36 peering, 69 active+clean; 475 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 19 MiB/s wr, 315 op/s 2024-03-29T04:59:26.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:26.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:59:27.347 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:27 smithi181 podman[115081]: 2024-03-29 04:59:27.035835247 +0000 UTC m=+0.573354619 container init 03d36b79ef26d0e5c3783209243761f73aed0cc0e7999558db8183342e0adb87 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, GIT_CLEAN=True) 2024-03-29T04:59:27.347 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:27 smithi181 podman[115081]: 2024-03-29 04:59:27.041306057 +0000 UTC m=+0.578825410 container start 03d36b79ef26d0e5c3783209243761f73aed0cc0e7999558db8183342e0adb87 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:59:27.347 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-osd[115119]: -- 172.21.15.181:0/49123491 <== mon.2 v2:172.21.15.181:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a6614ad680 con 0x55a661476c00 2024-03-29T04:59:27.347 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:27 smithi181 bash[115081]: 03d36b79ef26d0e5c3783209243761f73aed0cc0e7999558db8183342e0adb87 2024-03-29T04:59:27.347 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:27 smithi181 systemd[1]: Started Ceph osd.0 for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T04:59:27.372 INFO:tasks.workunit.client.0.smithi181.stdout:clone image263 -> image264 2024-03-29T04:59:27.452 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 123752518 2024-03-29T04:59:27.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:27 smithi187 ceph-mon[93536]: failed to encode map e419 with expected crc 2024-03-29T04:59:27.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:27 smithi187 ceph-mon[93536]: failed to encode map e419 with expected crc 2024-03-29T04:59:27.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:27 smithi187 ceph-mon[93536]: osdmap e419: 8 total, 7 up, 8 in 2024-03-29T04:59:27.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:27 smithi187 ceph-mon[93536]: failed to encode map e419 with expected crc 2024-03-29T04:59:27.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:27 smithi187 ceph-mon[93536]: failed to encode map e419 with expected crc 2024-03-29T04:59:27.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:27 smithi187 ceph-mon[93536]: failed to encode map e419 with expected crc 2024-03-29T04:59:27.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:27 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:27.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:27 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:27.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:27 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:59:27.703 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0[115096]: 2024-03-29T04:59:27.579+0000 7ffafc7467c0 -1 Falling back to public interface 2024-03-29T04:59:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[101970]: failed to encode map e419 with expected crc 2024-03-29T04:59:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[101970]: failed to encode map e419 with expected crc 2024-03-29T04:59:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[101970]: osdmap e419: 8 total, 7 up, 8 in 2024-03-29T04:59:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[101970]: failed to encode map e419 with expected crc 2024-03-29T04:59:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[101970]: failed to encode map e419 with expected crc 2024-03-29T04:59:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[101970]: failed to encode map e419 with expected crc 2024-03-29T04:59:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:59:27.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[106466]: failed to encode map e419 with expected crc 2024-03-29T04:59:27.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[106466]: failed to encode map e419 with expected crc 2024-03-29T04:59:27.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[106466]: osdmap e419: 8 total, 7 up, 8 in 2024-03-29T04:59:27.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[106466]: failed to encode map e419 with expected crc 2024-03-29T04:59:27.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[106466]: failed to encode map e419 with expected crc 2024-03-29T04:59:27.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[106466]: failed to encode map e419 with expected crc 2024-03-29T04:59:27.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:27.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:27.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:27 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:59:27.713 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.Stress_LargerDstObjSize (3850 ms) 2024-03-29T04:59:27.713 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.NoSnaps_SmallerDstObjSize 2024-03-29T04:59:27.952 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.NoSnaps_SmallerDstObjSize (239 ms) 2024-03-29T04:59:27.952 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.Snaps_SmallerDstObjSize 2024-03-29T04:59:28.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:28 smithi187 ceph-mon[93536]: failed to encode map e420 with expected crc 2024-03-29T04:59:28.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:28 smithi187 ceph-mon[93536]: failed to encode map e420 with expected crc 2024-03-29T04:59:28.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:28 smithi187 ceph-mon[93536]: osdmap e420: 8 total, 7 up, 8 in 2024-03-29T04:59:28.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:28 smithi187 ceph-mon[93536]: failed to encode map e420 with expected crc 2024-03-29T04:59:28.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:28 smithi187 ceph-mon[93536]: failed to encode map e420 with expected crc 2024-03-29T04:59:28.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:28 smithi187 ceph-mon[93536]: failed to encode map e420 with expected crc 2024-03-29T04:59:28.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:28 smithi187 ceph-mon[93536]: pgmap v286: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 676 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 47 MiB/s wr, 480 op/s; 149/1503 objects degraded (9.914%) 2024-03-29T04:59:28.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:28 smithi181 ceph-mon[106466]: failed to encode map e420 with expected crc 2024-03-29T04:59:28.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:28 smithi181 ceph-mon[106466]: failed to encode map e420 with expected crc 2024-03-29T04:59:28.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:28 smithi181 ceph-mon[106466]: osdmap e420: 8 total, 7 up, 8 in 2024-03-29T04:59:28.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:28 smithi181 ceph-mon[106466]: failed to encode map e420 with expected crc 2024-03-29T04:59:28.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:28 smithi181 ceph-mon[106466]: failed to encode map e420 with expected crc 2024-03-29T04:59:28.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:28 smithi181 ceph-mon[106466]: failed to encode map e420 with expected crc 2024-03-29T04:59:28.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:28 smithi181 ceph-mon[106466]: pgmap v286: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 676 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 47 MiB/s wr, 480 op/s; 149/1503 objects degraded (9.914%) 2024-03-29T04:59:28.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:28 smithi181 ceph-mon[101970]: failed to encode map e420 with expected crc 2024-03-29T04:59:28.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:28 smithi181 ceph-mon[101970]: failed to encode map e420 with expected crc 2024-03-29T04:59:28.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:28 smithi181 ceph-mon[101970]: osdmap e420: 8 total, 7 up, 8 in 2024-03-29T04:59:28.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:28 smithi181 ceph-mon[101970]: failed to encode map e420 with expected crc 2024-03-29T04:59:28.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:28 smithi181 ceph-mon[101970]: failed to encode map e420 with expected crc 2024-03-29T04:59:28.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:28 smithi181 ceph-mon[101970]: failed to encode map e420 with expected crc 2024-03-29T04:59:28.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:28 smithi181 ceph-mon[101970]: pgmap v286: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 676 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 47 MiB/s wr, 480 op/s; 149/1503 objects degraded (9.914%) 2024-03-29T04:59:29.030 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:59:28 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:59:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:59:29.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:59:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:59:28] "GET /metrics HTTP/1.1" 200 36429 "" "Prometheus/2.43.0" 2024-03-29T04:59:29.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:29 smithi187 ceph-mon[93536]: Health check failed: Degraded data redundancy: 149/1503 objects degraded (9.914%), 17 pgs degraded (PG_DEGRADED) 2024-03-29T04:59:29.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:29 smithi187 ceph-mon[93536]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 11 pgs peering) 2024-03-29T04:59:29.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:29 smithi187 ceph-mon[93536]: failed to encode map e421 with expected crc 2024-03-29T04:59:29.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:29 smithi187 ceph-mon[93536]: failed to encode map e421 with expected crc 2024-03-29T04:59:29.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:29 smithi187 ceph-mon[93536]: osdmap e421: 8 total, 7 up, 8 in 2024-03-29T04:59:29.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:29 smithi187 ceph-mon[93536]: failed to encode map e421 with expected crc 2024-03-29T04:59:29.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:29 smithi181 ceph-mon[101970]: Health check failed: Degraded data redundancy: 149/1503 objects degraded (9.914%), 17 pgs degraded (PG_DEGRADED) 2024-03-29T04:59:29.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:29 smithi181 ceph-mon[101970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 11 pgs peering) 2024-03-29T04:59:29.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:29 smithi181 ceph-mon[101970]: failed to encode map e421 with expected crc 2024-03-29T04:59:29.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:29 smithi181 ceph-mon[101970]: failed to encode map e421 with expected crc 2024-03-29T04:59:29.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:29 smithi181 ceph-mon[101970]: osdmap e421: 8 total, 7 up, 8 in 2024-03-29T04:59:29.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:29 smithi181 ceph-mon[101970]: failed to encode map e421 with expected crc 2024-03-29T04:59:29.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:29 smithi181 ceph-mon[106466]: Health check failed: Degraded data redundancy: 149/1503 objects degraded (9.914%), 17 pgs degraded (PG_DEGRADED) 2024-03-29T04:59:29.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:29 smithi181 ceph-mon[106466]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 11 pgs peering) 2024-03-29T04:59:29.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:29 smithi181 ceph-mon[106466]: failed to encode map e421 with expected crc 2024-03-29T04:59:29.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:29 smithi181 ceph-mon[106466]: failed to encode map e421 with expected crc 2024-03-29T04:59:29.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:29 smithi181 ceph-mon[106466]: osdmap e421: 8 total, 7 up, 8 in 2024-03-29T04:59:29.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:29 smithi181 ceph-mon[106466]: failed to encode map e421 with expected crc 2024-03-29T04:59:30.633 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0[115096]: 2024-03-29T04:59:30.396+0000 7ffafc7467c0 -1 osd.0 413 log_to_monitors true 2024-03-29T04:59:30.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:30 smithi181 ceph-mon[101970]: failed to encode map e422 with expected crc 2024-03-29T04:59:30.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:30 smithi181 ceph-mon[101970]: failed to encode map e422 with expected crc 2024-03-29T04:59:30.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:30 smithi181 ceph-mon[101970]: osdmap e422: 8 total, 7 up, 8 in 2024-03-29T04:59:30.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:30 smithi181 ceph-mon[101970]: failed to encode map e422 with expected crc 2024-03-29T04:59:30.634 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:30 smithi181 ceph-mon[101970]: pgmap v289: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 676 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 47 MiB/s wr, 480 op/s; 149/1503 objects degraded (9.914%) 2024-03-29T04:59:30.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:30 smithi181 ceph-mon[106466]: failed to encode map e422 with expected crc 2024-03-29T04:59:30.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:30 smithi181 ceph-mon[106466]: failed to encode map e422 with expected crc 2024-03-29T04:59:30.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:30 smithi181 ceph-mon[106466]: osdmap e422: 8 total, 7 up, 8 in 2024-03-29T04:59:30.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:30 smithi181 ceph-mon[106466]: failed to encode map e422 with expected crc 2024-03-29T04:59:30.635 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:30 smithi181 ceph-mon[106466]: pgmap v289: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 676 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 47 MiB/s wr, 480 op/s; 149/1503 objects degraded (9.914%) 2024-03-29T04:59:30.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:30 smithi187 ceph-mon[93536]: failed to encode map e422 with expected crc 2024-03-29T04:59:30.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:30 smithi187 ceph-mon[93536]: failed to encode map e422 with expected crc 2024-03-29T04:59:30.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:30 smithi187 ceph-mon[93536]: osdmap e422: 8 total, 7 up, 8 in 2024-03-29T04:59:30.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:30 smithi187 ceph-mon[93536]: failed to encode map e422 with expected crc 2024-03-29T04:59:30.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:30 smithi187 ceph-mon[93536]: pgmap v289: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 676 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 47 MiB/s wr, 480 op/s; 149/1503 objects degraded (9.914%) 2024-03-29T04:59:31.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:31 smithi187 ceph-mon[93536]: failed to encode map e423 with expected crc 2024-03-29T04:59:31.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:31 smithi187 ceph-mon[93536]: failed to encode map e423 with expected crc 2024-03-29T04:59:31.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:31 smithi187 ceph-mon[93536]: osdmap e423: 8 total, 7 up, 8 in 2024-03-29T04:59:31.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:31 smithi187 ceph-mon[93536]: failed to encode map e423 with expected crc 2024-03-29T04:59:31.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:31 smithi187 ceph-mon[93536]: failed to encode map e423 with expected crc 2024-03-29T04:59:31.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:31 smithi187 ceph-mon[93536]: from='osd.0 [v2:172.21.15.181:6802/849347250,v1:172.21.15.181:6803/849347250]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:59:31.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:31 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:31.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:31 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:31.659 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.Snaps_SmallerDstObjSize (3707 ms) 2024-03-29T04:59:31.659 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.Clone_SmallerDstObjSize 2024-03-29T04:59:31.704 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0[115096]: 2024-03-29T04:59:31.398+0000 7ffaedb9e700 -1 osd.0 413 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:59:31.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-mon[106466]: failed to encode map e423 with expected crc 2024-03-29T04:59:31.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-mon[106466]: failed to encode map e423 with expected crc 2024-03-29T04:59:31.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-mon[106466]: osdmap e423: 8 total, 7 up, 8 in 2024-03-29T04:59:31.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-mon[106466]: failed to encode map e423 with expected crc 2024-03-29T04:59:31.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-mon[106466]: failed to encode map e423 with expected crc 2024-03-29T04:59:31.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-mon[106466]: from='osd.0 [v2:172.21.15.181:6802/849347250,v1:172.21.15.181:6803/849347250]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:59:31.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:31.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:31.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-mon[101970]: failed to encode map e423 with expected crc 2024-03-29T04:59:31.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-mon[101970]: failed to encode map e423 with expected crc 2024-03-29T04:59:31.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-mon[101970]: osdmap e423: 8 total, 7 up, 8 in 2024-03-29T04:59:31.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-mon[101970]: failed to encode map e423 with expected crc 2024-03-29T04:59:31.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-mon[101970]: failed to encode map e423 with expected crc 2024-03-29T04:59:31.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-mon[101970]: from='osd.0 [v2:172.21.15.181:6802/849347250,v1:172.21.15.181:6803/849347250]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:59:31.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:31.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:31 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:32.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[106466]: from='osd.0 [v2:172.21.15.181:6802/849347250,v1:172.21.15.181:6803/849347250]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:59:32.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[106466]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[106466]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[106466]: osdmap e424: 8 total, 7 up, 8 in 2024-03-29T04:59:32.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[106466]: from='osd.0 [v2:172.21.15.181:6802/849347250,v1:172.21.15.181:6803/849347250]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:59:32.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[106466]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[106466]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[106466]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[106466]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:32.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:32.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[106466]: pgmap v292: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 676 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 149/1503 objects degraded (9.914%) 2024-03-29T04:59:32.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:32.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:32.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[101970]: from='osd.0 [v2:172.21.15.181:6802/849347250,v1:172.21.15.181:6803/849347250]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:59:32.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[101970]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[101970]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[101970]: osdmap e424: 8 total, 7 up, 8 in 2024-03-29T04:59:32.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[101970]: from='osd.0 [v2:172.21.15.181:6802/849347250,v1:172.21.15.181:6803/849347250]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:59:32.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[101970]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[101970]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[101970]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[101970]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:32.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:32.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[101970]: pgmap v292: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 676 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 149/1503 objects degraded (9.914%) 2024-03-29T04:59:32.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:32.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:32 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:32.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:32 smithi187 ceph-mon[93536]: from='osd.0 [v2:172.21.15.181:6802/849347250,v1:172.21.15.181:6803/849347250]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:59:32.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:32 smithi187 ceph-mon[93536]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:32 smithi187 ceph-mon[93536]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:32 smithi187 ceph-mon[93536]: osdmap e424: 8 total, 7 up, 8 in 2024-03-29T04:59:32.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:32 smithi187 ceph-mon[93536]: from='osd.0 [v2:172.21.15.181:6802/849347250,v1:172.21.15.181:6803/849347250]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T04:59:32.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:32 smithi187 ceph-mon[93536]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:32 smithi187 ceph-mon[93536]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:32 smithi187 ceph-mon[93536]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:32 smithi187 ceph-mon[93536]: failed to encode map e424 with expected crc 2024-03-29T04:59:32.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:32 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:32.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:32 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:32.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:32 smithi187 ceph-mon[93536]: pgmap v292: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 676 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 149/1503 objects degraded (9.914%) 2024-03-29T04:59:32.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:32 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:32.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:32 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:33.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:33 smithi181 ceph-mon[101970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:59:33.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:33 smithi181 ceph-mon[101970]: failed to encode map e425 with expected crc 2024-03-29T04:59:33.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:33 smithi181 ceph-mon[101970]: failed to encode map e425 with expected crc 2024-03-29T04:59:33.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:33 smithi181 ceph-mon[101970]: osd.0 [v2:172.21.15.181:6802/849347250,v1:172.21.15.181:6803/849347250] boot 2024-03-29T04:59:33.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:33 smithi181 ceph-mon[101970]: osdmap e425: 8 total, 8 up, 8 in 2024-03-29T04:59:33.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:33 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:59:33.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:33 smithi181 ceph-mon[101970]: failed to encode map e425 with expected crc 2024-03-29T04:59:33.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:33 smithi181 ceph-mon[101970]: osdmap e426: 8 total, 8 up, 8 in 2024-03-29T04:59:33.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:33 smithi181 ceph-mon[106466]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:59:33.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:33 smithi181 ceph-mon[106466]: failed to encode map e425 with expected crc 2024-03-29T04:59:33.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:33 smithi181 ceph-mon[106466]: failed to encode map e425 with expected crc 2024-03-29T04:59:33.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:33 smithi181 ceph-mon[106466]: osd.0 [v2:172.21.15.181:6802/849347250,v1:172.21.15.181:6803/849347250] boot 2024-03-29T04:59:33.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:33 smithi181 ceph-mon[106466]: osdmap e425: 8 total, 8 up, 8 in 2024-03-29T04:59:33.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:33 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:59:33.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:33 smithi181 ceph-mon[106466]: failed to encode map e425 with expected crc 2024-03-29T04:59:33.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:33 smithi181 ceph-mon[106466]: osdmap e426: 8 total, 8 up, 8 in 2024-03-29T04:59:33.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:33 smithi187 ceph-mon[93536]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:59:33.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:33 smithi187 ceph-mon[93536]: failed to encode map e425 with expected crc 2024-03-29T04:59:33.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:33 smithi187 ceph-mon[93536]: failed to encode map e425 with expected crc 2024-03-29T04:59:33.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:33 smithi187 ceph-mon[93536]: osd.0 [v2:172.21.15.181:6802/849347250,v1:172.21.15.181:6803/849347250] boot 2024-03-29T04:59:33.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:33 smithi187 ceph-mon[93536]: osdmap e425: 8 total, 8 up, 8 in 2024-03-29T04:59:33.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:33 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:59:33.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:33 smithi187 ceph-mon[93536]: failed to encode map e425 with expected crc 2024-03-29T04:59:33.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:33 smithi187 ceph-mon[93536]: osdmap e426: 8 total, 8 up, 8 in 2024-03-29T04:59:34.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[106466]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[106466]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[106466]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[106466]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[106466]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[106466]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[106466]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[106466]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[106466]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[106466]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[106466]: pgmap v295: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 762 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 53 MiB/s rd, 21 MiB/s wr, 505 op/s; 175/1746 objects degraded (10.023%) 2024-03-29T04:59:34.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[101970]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[101970]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[101970]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[101970]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[101970]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[101970]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[101970]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[101970]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[101970]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[101970]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:34 smithi181 ceph-mon[101970]: pgmap v295: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 762 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 53 MiB/s rd, 21 MiB/s wr, 505 op/s; 175/1746 objects degraded (10.023%) 2024-03-29T04:59:34.727 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.Clone_SmallerDstObjSize (3068 ms) 2024-03-29T04:59:34.727 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.CloneFlatten_SmallerDstObjSize 2024-03-29T04:59:34.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:34 smithi187 ceph-mon[93536]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:34 smithi187 ceph-mon[93536]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:34 smithi187 ceph-mon[93536]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:34 smithi187 ceph-mon[93536]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:34 smithi187 ceph-mon[93536]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:34 smithi187 ceph-mon[93536]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:34 smithi187 ceph-mon[93536]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:34 smithi187 ceph-mon[93536]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:34 smithi187 ceph-mon[93536]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:34 smithi187 ceph-mon[93536]: failed to encode map e426 with expected crc 2024-03-29T04:59:34.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:34 smithi187 ceph-mon[93536]: pgmap v295: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 762 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 53 MiB/s rd, 21 MiB/s wr, 505 op/s; 175/1746 objects degraded (10.023%) 2024-03-29T04:59:35.670 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:35 smithi181 ceph-mon[106466]: Health check update: Degraded data redundancy: 175/1746 objects degraded (10.023%), 17 pgs degraded (PG_DEGRADED) 2024-03-29T04:59:35.671 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:35 smithi181 ceph-mon[106466]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.671 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:35 smithi181 ceph-mon[106466]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.671 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:35 smithi181 ceph-mon[106466]: osdmap e427: 8 total, 8 up, 8 in 2024-03-29T04:59:35.671 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:35 smithi181 ceph-mon[106466]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.671 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:35 smithi181 ceph-mon[106466]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.671 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:35 smithi181 ceph-mon[106466]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.671 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:35 smithi181 ceph-mon[106466]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.671 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:35 smithi181 ceph-mon[101970]: Health check update: Degraded data redundancy: 175/1746 objects degraded (10.023%), 17 pgs degraded (PG_DEGRADED) 2024-03-29T04:59:35.671 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:35 smithi181 ceph-mon[101970]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.672 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:35 smithi181 ceph-mon[101970]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.672 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:35 smithi181 ceph-mon[101970]: osdmap e427: 8 total, 8 up, 8 in 2024-03-29T04:59:35.672 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:35 smithi181 ceph-mon[101970]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.672 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:35 smithi181 ceph-mon[101970]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.672 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:35 smithi181 ceph-mon[101970]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.672 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:35 smithi181 ceph-mon[101970]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:35 smithi187 ceph-mon[93536]: Health check update: Degraded data redundancy: 175/1746 objects degraded (10.023%), 17 pgs degraded (PG_DEGRADED) 2024-03-29T04:59:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:35 smithi187 ceph-mon[93536]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:35 smithi187 ceph-mon[93536]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:35 smithi187 ceph-mon[93536]: osdmap e427: 8 total, 8 up, 8 in 2024-03-29T04:59:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:35 smithi187 ceph-mon[93536]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:35 smithi187 ceph-mon[93536]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:35 smithi187 ceph-mon[93536]: failed to encode map e427 with expected crc 2024-03-29T04:59:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:35 smithi187 ceph-mon[93536]: failed to encode map e427 with expected crc 2024-03-29T04:59:36.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: failed to encode map e428 with expected crc 2024-03-29T04:59:36.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: failed to encode map e428 with expected crc 2024-03-29T04:59:36.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: osdmap e428: 8 total, 8 up, 8 in 2024-03-29T04:59:36.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: failed to encode map e428 with expected crc 2024-03-29T04:59:36.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: failed to encode map e428 with expected crc 2024-03-29T04:59:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: failed to encode map e428 with expected crc 2024-03-29T04:59:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:59:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:59:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: pgmap v298: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 762 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 21 MiB/s wr, 510 op/s; 175/1746 objects degraded (10.023%) 2024-03-29T04:59:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: pgmap v299: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 762 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 175/1746 objects degraded (10.023%) 2024-03-29T04:59:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:59:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:59:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:59:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: pgmap v300: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 762 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 175/1746 objects degraded (10.023%) 2024-03-29T04:59:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: pgmap v301: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 762 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 175/1746 objects degraded (10.023%) 2024-03-29T04:59:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:36.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: failed to encode map e428 with expected crc 2024-03-29T04:59:36.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: failed to encode map e428 with expected crc 2024-03-29T04:59:36.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: osdmap e428: 8 total, 8 up, 8 in 2024-03-29T04:59:36.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: failed to encode map e428 with expected crc 2024-03-29T04:59:36.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: failed to encode map e428 with expected crc 2024-03-29T04:59:36.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: failed to encode map e428 with expected crc 2024-03-29T04:59:36.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:36.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:36.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:59:36.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:59:36.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: pgmap v298: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 762 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 21 MiB/s wr, 510 op/s; 175/1746 objects degraded (10.023%) 2024-03-29T04:59:36.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: pgmap v299: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 762 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 175/1746 objects degraded (10.023%) 2024-03-29T04:59:36.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:36.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:59:36.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:59:36.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:59:36.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: pgmap v300: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 762 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 175/1746 objects degraded (10.023%) 2024-03-29T04:59:36.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: pgmap v301: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 762 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 175/1746 objects degraded (10.023%) 2024-03-29T04:59:36.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:36.880 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.CloneFlatten_SmallerDstObjSize (2153 ms) 2024-03-29T04:59:36.880 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.Stress_SmallerDstObjSize 2024-03-29T04:59:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: failed to encode map e428 with expected crc 2024-03-29T04:59:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: failed to encode map e428 with expected crc 2024-03-29T04:59:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: osdmap e428: 8 total, 8 up, 8 in 2024-03-29T04:59:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: failed to encode map e428 with expected crc 2024-03-29T04:59:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: failed to encode map e428 with expected crc 2024-03-29T04:59:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: failed to encode map e428 with expected crc 2024-03-29T04:59:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:59:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:59:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: pgmap v298: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 762 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 54 MiB/s rd, 21 MiB/s wr, 510 op/s; 175/1746 objects degraded (10.023%) 2024-03-29T04:59:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: pgmap v299: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 762 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 175/1746 objects degraded (10.023%) 2024-03-29T04:59:36.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:36.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:59:36.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:59:36.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:59:36.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: pgmap v300: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 762 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 175/1746 objects degraded (10.023%) 2024-03-29T04:59:36.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: pgmap v301: 105 pgs: 17 active+undersized+degraded, 19 active+undersized, 69 active+clean; 762 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 175/1746 objects degraded (10.023%) 2024-03-29T04:59:36.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:36.902 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 79108615~4777465 2024-03-29T04:59:36.920 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 52442961~4180020 2024-03-29T04:59:36.921 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 77569294~5958629 2024-03-29T04:59:36.938 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 2303444~7996965 2024-03-29T04:59:36.939 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 30250375~7064487 2024-03-29T04:59:36.962 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 45038793~5412286 2024-03-29T04:59:36.966 INFO:tasks.workunit.client.0.smithi181.stdout:write: A 53651004~2978817 2024-03-29T04:59:36.971 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 61427089~1710382 2024-03-29T04:59:36.971 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap0 2024-03-29T04:59:37.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:37 smithi181 ceph-mon[106466]: failed to encode map e429 with expected crc 2024-03-29T04:59:37.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:37 smithi181 ceph-mon[106466]: failed to encode map e429 with expected crc 2024-03-29T04:59:37.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:37 smithi181 ceph-mon[106466]: osdmap e429: 8 total, 8 up, 8 in 2024-03-29T04:59:37.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:37 smithi181 ceph-mon[106466]: failed to encode map e429 with expected crc 2024-03-29T04:59:37.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:37 smithi181 ceph-mon[106466]: failed to encode map e429 with expected crc 2024-03-29T04:59:37.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:37 smithi181 ceph-mon[106466]: failed to encode map e429 with expected crc 2024-03-29T04:59:37.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:37 smithi181 ceph-mon[101970]: failed to encode map e429 with expected crc 2024-03-29T04:59:37.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:37 smithi181 ceph-mon[101970]: failed to encode map e429 with expected crc 2024-03-29T04:59:37.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:37 smithi181 ceph-mon[101970]: osdmap e429: 8 total, 8 up, 8 in 2024-03-29T04:59:37.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:37 smithi181 ceph-mon[101970]: failed to encode map e429 with expected crc 2024-03-29T04:59:37.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:37 smithi181 ceph-mon[101970]: failed to encode map e429 with expected crc 2024-03-29T04:59:37.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:37 smithi181 ceph-mon[101970]: failed to encode map e429 with expected crc 2024-03-29T04:59:37.768 INFO:tasks.workunit.client.0.smithi181.stdout:clone image283 -> image284 2024-03-29T04:59:37.822 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 16960179~1058370 2024-03-29T04:59:37.830 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 66163859~2392841 2024-03-29T04:59:37.834 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 35469432~5633574 2024-03-29T04:59:37.867 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 18866055~8046833 2024-03-29T04:59:37.871 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 43780948~3418731 2024-03-29T04:59:37.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:37 smithi187 ceph-mon[93536]: failed to encode map e429 with expected crc 2024-03-29T04:59:37.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:37 smithi187 ceph-mon[93536]: failed to encode map e429 with expected crc 2024-03-29T04:59:37.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:37 smithi187 ceph-mon[93536]: osdmap e429: 8 total, 8 up, 8 in 2024-03-29T04:59:37.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:37 smithi187 ceph-mon[93536]: failed to encode map e429 with expected crc 2024-03-29T04:59:37.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:37 smithi187 ceph-mon[93536]: failed to encode map e429 with expected crc 2024-03-29T04:59:37.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:37 smithi187 ceph-mon[93536]: failed to encode map e429 with expected crc 2024-03-29T04:59:37.884 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 28299850~8260522 2024-03-29T04:59:37.886 INFO:tasks.workunit.client.0.smithi181.stdout:write: B 21426595~945849 2024-03-29T04:59:37.890 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 21903085~7305996 2024-03-29T04:59:37.892 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap1 2024-03-29T04:59:38.765 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 99472379 2024-03-29T04:59:38.776 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 39438598~6369049 2024-03-29T04:59:38.779 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 92905658~5873570 2024-03-29T04:59:38.780 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 18496659~4203139 2024-03-29T04:59:38.780 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 69763313~6043810 2024-03-29T04:59:38.781 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 24232565~2291097 2024-03-29T04:59:38.781 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 67734495~7275061 2024-03-29T04:59:38.815 INFO:tasks.workunit.client.0.smithi181.stdout:write: C 86423870~1420406 2024-03-29T04:59:38.815 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 8707442~688827 2024-03-29T04:59:38.827 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap2 2024-03-29T04:59:39.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:38 smithi187 ceph-mon[93536]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:38 smithi187 ceph-mon[93536]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:38 smithi187 ceph-mon[93536]: osdmap e430: 8 total, 8 up, 8 in 2024-03-29T04:59:39.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:38 smithi187 ceph-mon[93536]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:38 smithi187 ceph-mon[93536]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:38 smithi187 ceph-mon[93536]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:38 smithi187 ceph-mon[93536]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:38 smithi187 ceph-mon[93536]: pgmap v304: 105 pgs: 105 active+clean; 905 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 82 MiB/s rd, 66 MiB/s wr, 1.06k op/s; 41 MiB/s, 26 keys/s, 23 objects/s recovering 2024-03-29T04:59:39.136 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:59:38 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:59:38.660Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-03-29T04:59:39.136 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 04:59:38 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T04:59:38.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi187.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.181\", device_class=\"ssd\", hostname=\"smithi181\", instance=\"smithi181:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.181\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:59:39.203 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:59:38] "GET /metrics HTTP/1.1" 200 36429 "" "Prometheus/2.43.0" 2024-03-29T04:59:39.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-mon[106466]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-mon[106466]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-mon[106466]: osdmap e430: 8 total, 8 up, 8 in 2024-03-29T04:59:39.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-mon[106466]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-mon[106466]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-mon[106466]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-mon[106466]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-mon[106466]: pgmap v304: 105 pgs: 105 active+clean; 905 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 82 MiB/s rd, 66 MiB/s wr, 1.06k op/s; 41 MiB/s, 26 keys/s, 23 objects/s recovering 2024-03-29T04:59:39.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-mon[101970]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-mon[101970]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-mon[101970]: osdmap e430: 8 total, 8 up, 8 in 2024-03-29T04:59:39.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-mon[101970]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-mon[101970]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-mon[101970]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-mon[101970]: failed to encode map e430 with expected crc 2024-03-29T04:59:39.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:38 smithi181 ceph-mon[101970]: pgmap v304: 105 pgs: 105 active+clean; 905 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 82 MiB/s rd, 66 MiB/s wr, 1.06k op/s; 41 MiB/s, 26 keys/s, 23 objects/s recovering 2024-03-29T04:59:39.781 INFO:tasks.workunit.client.0.smithi181.stdout:clone image284 -> image285 2024-03-29T04:59:39.851 INFO:tasks.workunit.client.0.smithi181.stdout:resize: 173808183 2024-03-29T04:59:39.861 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 94304688~6791831 2024-03-29T04:59:39.883 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 42261641~1971535 2024-03-29T04:59:39.899 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 8644286~767977 2024-03-29T04:59:39.904 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 114464171~2924516 2024-03-29T04:59:39.905 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 16742963~8043253 2024-03-29T04:59:39.930 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 163574182~4561382 2024-03-29T04:59:39.931 INFO:tasks.workunit.client.0.smithi181.stdout:write: D 138591695~7665650 2024-03-29T04:59:39.960 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 162438198~4337298 2024-03-29T04:59:39.960 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap3 2024-03-29T04:59:40.103 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:39 smithi187 ceph-mon[93536]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 175/1746 objects degraded (10.023%), 17 pgs degraded) 2024-03-29T04:59:40.103 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:39 smithi187 ceph-mon[93536]: failed to encode map e431 with expected crc 2024-03-29T04:59:40.103 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:39 smithi187 ceph-mon[93536]: failed to encode map e431 with expected crc 2024-03-29T04:59:40.103 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:39 smithi187 ceph-mon[93536]: osdmap e431: 8 total, 8 up, 8 in 2024-03-29T04:59:40.103 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:39 smithi187 ceph-mon[93536]: failed to encode map e431 with expected crc 2024-03-29T04:59:40.104 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:39 smithi187 ceph-mon[93536]: failed to encode map e431 with expected crc 2024-03-29T04:59:40.104 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:39 smithi187 ceph-mon[93536]: failed to encode map e431 with expected crc 2024-03-29T04:59:40.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:39 smithi181 ceph-mon[101970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 175/1746 objects degraded (10.023%), 17 pgs degraded) 2024-03-29T04:59:40.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:39 smithi181 ceph-mon[101970]: failed to encode map e431 with expected crc 2024-03-29T04:59:40.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:39 smithi181 ceph-mon[101970]: failed to encode map e431 with expected crc 2024-03-29T04:59:40.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:39 smithi181 ceph-mon[101970]: osdmap e431: 8 total, 8 up, 8 in 2024-03-29T04:59:40.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:39 smithi181 ceph-mon[101970]: failed to encode map e431 with expected crc 2024-03-29T04:59:40.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:39 smithi181 ceph-mon[101970]: failed to encode map e431 with expected crc 2024-03-29T04:59:40.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:39 smithi181 ceph-mon[101970]: failed to encode map e431 with expected crc 2024-03-29T04:59:40.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:39 smithi181 ceph-mon[106466]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 175/1746 objects degraded (10.023%), 17 pgs degraded) 2024-03-29T04:59:40.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:39 smithi181 ceph-mon[106466]: failed to encode map e431 with expected crc 2024-03-29T04:59:40.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:39 smithi181 ceph-mon[106466]: failed to encode map e431 with expected crc 2024-03-29T04:59:40.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:39 smithi181 ceph-mon[106466]: osdmap e431: 8 total, 8 up, 8 in 2024-03-29T04:59:40.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:39 smithi181 ceph-mon[106466]: failed to encode map e431 with expected crc 2024-03-29T04:59:40.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:39 smithi181 ceph-mon[106466]: failed to encode map e431 with expected crc 2024-03-29T04:59:40.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:39 smithi181 ceph-mon[106466]: failed to encode map e431 with expected crc 2024-03-29T04:59:40.776 INFO:tasks.workunit.client.0.smithi181.stdout:clone image285 -> image286 2024-03-29T04:59:40.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:40 smithi181 ceph-mon[101970]: failed to encode map e432 with expected crc 2024-03-29T04:59:40.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:40 smithi181 ceph-mon[101970]: failed to encode map e432 with expected crc 2024-03-29T04:59:40.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:40 smithi181 ceph-mon[101970]: osdmap e432: 8 total, 8 up, 8 in 2024-03-29T04:59:40.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:40 smithi181 ceph-mon[101970]: failed to encode map e432 with expected crc 2024-03-29T04:59:40.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:40 smithi181 ceph-mon[101970]: failed to encode map e432 with expected crc 2024-03-29T04:59:40.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:40 smithi181 ceph-mon[101970]: pgmap v307: 105 pgs: 105 active+clean; 905 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 44 MiB/s rd, 35 MiB/s wr, 566 op/s; 22 MiB/s, 14 keys/s, 12 objects/s recovering 2024-03-29T04:59:40.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:40 smithi181 ceph-mon[106466]: failed to encode map e432 with expected crc 2024-03-29T04:59:40.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:40 smithi181 ceph-mon[106466]: failed to encode map e432 with expected crc 2024-03-29T04:59:40.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:40 smithi181 ceph-mon[106466]: osdmap e432: 8 total, 8 up, 8 in 2024-03-29T04:59:40.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:40 smithi181 ceph-mon[106466]: failed to encode map e432 with expected crc 2024-03-29T04:59:40.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:40 smithi181 ceph-mon[106466]: failed to encode map e432 with expected crc 2024-03-29T04:59:40.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:40 smithi181 ceph-mon[106466]: pgmap v307: 105 pgs: 105 active+clean; 905 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 44 MiB/s rd, 35 MiB/s wr, 566 op/s; 22 MiB/s, 14 keys/s, 12 objects/s recovering 2024-03-29T04:59:41.127 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.Stress_SmallerDstObjSize (4247 ms) 2024-03-29T04:59:41.127 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize 2024-03-29T04:59:41.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:40 smithi187 ceph-mon[93536]: failed to encode map e432 with expected crc 2024-03-29T04:59:41.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:40 smithi187 ceph-mon[93536]: failed to encode map e432 with expected crc 2024-03-29T04:59:41.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:40 smithi187 ceph-mon[93536]: osdmap e432: 8 total, 8 up, 8 in 2024-03-29T04:59:41.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:40 smithi187 ceph-mon[93536]: failed to encode map e432 with expected crc 2024-03-29T04:59:41.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:40 smithi187 ceph-mon[93536]: failed to encode map e432 with expected crc 2024-03-29T04:59:41.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:40 smithi187 ceph-mon[93536]: pgmap v307: 105 pgs: 105 active+clean; 905 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 44 MiB/s rd, 35 MiB/s wr, 566 op/s; 22 MiB/s, 14 keys/s, 12 objects/s recovering 2024-03-29T04:59:41.149 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:59:41.221 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize (94 ms) 2024-03-29T04:59:41.221 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingLargerDstObjSize 2024-03-29T04:59:41.243 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:59:41.317 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.Snaps_StrippingLargerDstObjSize (96 ms) 2024-03-29T04:59:41.318 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.Clone_StrippingLargerDstObjSize 2024-03-29T04:59:41.338 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:59:41.413 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.Clone_StrippingLargerDstObjSize (95 ms) 2024-03-29T04:59:41.413 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize 2024-03-29T04:59:41.432 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:59:41.502 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize (89 ms) 2024-03-29T04:59:41.502 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.Stress_StrippingLargerDstObjSize 2024-03-29T04:59:41.526 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:59:41.603 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.Stress_StrippingLargerDstObjSize (98 ms) 2024-03-29T04:59:41.604 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize 2024-03-29T04:59:41.620 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:59:41.690 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize (91 ms) 2024-03-29T04:59:41.691 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize 2024-03-29T04:59:41.712 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:59:41.784 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize (93 ms) 2024-03-29T04:59:41.784 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.Clone_StrippingSmallerDstObjSize 2024-03-29T04:59:41.805 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:59:41.878 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.Clone_StrippingSmallerDstObjSize (93 ms) 2024-03-29T04:59:41.878 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize 2024-03-29T04:59:41.899 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:59:41.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:41 smithi181 ceph-mon[106466]: failed to encode map e433 with expected crc 2024-03-29T04:59:41.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:41 smithi181 ceph-mon[106466]: failed to encode map e433 with expected crc 2024-03-29T04:59:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:41 smithi181 ceph-mon[106466]: osdmap e433: 8 total, 8 up, 8 in 2024-03-29T04:59:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:41 smithi181 ceph-mon[106466]: failed to encode map e433 with expected crc 2024-03-29T04:59:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:59:41.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:41 smithi181 ceph-mon[101970]: failed to encode map e433 with expected crc 2024-03-29T04:59:41.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:41 smithi181 ceph-mon[101970]: failed to encode map e433 with expected crc 2024-03-29T04:59:41.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:41 smithi181 ceph-mon[101970]: osdmap e433: 8 total, 8 up, 8 in 2024-03-29T04:59:41.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:41 smithi181 ceph-mon[101970]: failed to encode map e433 with expected crc 2024-03-29T04:59:41.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:41.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:59:41.971 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize (95 ms) 2024-03-29T04:59:41.972 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestDeepCopy.Stress_StrippingSmallerDstObjSize 2024-03-29T04:59:41.994 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:59:42.071 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestDeepCopy.Stress_StrippingSmallerDstObjSize (100 ms) 2024-03-29T04:59:42.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:41 smithi187 ceph-mon[93536]: failed to encode map e433 with expected crc 2024-03-29T04:59:42.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:41 smithi187 ceph-mon[93536]: failed to encode map e433 with expected crc 2024-03-29T04:59:42.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:41 smithi187 ceph-mon[93536]: osdmap e433: 8 total, 8 up, 8 in 2024-03-29T04:59:42.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:41 smithi187 ceph-mon[93536]: failed to encode map e433 with expected crc 2024-03-29T04:59:42.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:42.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:59:42.774 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 31 tests from TestDeepCopy (55465 ms total) 2024-03-29T04:59:42.774 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T04:59:42.774 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 51 tests from TestMirroring 2024-03-29T04:59:43.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:42 smithi187 ceph-mon[93536]: pgmap v309: 105 pgs: 105 active+clean; 915 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-03-29T04:59:43.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:42 smithi181 ceph-mon[101970]: pgmap v309: 105 pgs: 105 active+clean; 915 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-03-29T04:59:43.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:42 smithi181 ceph-mon[106466]: pgmap v309: 105 pgs: 105 active+clean; 915 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-03-29T04:59:44.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:43 smithi187 ceph-mon[93536]: failed to encode map e434 with expected crc 2024-03-29T04:59:44.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:43 smithi187 ceph-mon[93536]: failed to encode map e434 with expected crc 2024-03-29T04:59:44.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:43 smithi187 ceph-mon[93536]: osdmap e434: 8 total, 8 up, 8 in 2024-03-29T04:59:44.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:43 smithi187 ceph-mon[93536]: failed to encode map e434 with expected crc 2024-03-29T04:59:44.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:43 smithi181 ceph-mon[106466]: failed to encode map e434 with expected crc 2024-03-29T04:59:44.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:43 smithi181 ceph-mon[106466]: failed to encode map e434 with expected crc 2024-03-29T04:59:44.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:43 smithi181 ceph-mon[106466]: osdmap e434: 8 total, 8 up, 8 in 2024-03-29T04:59:44.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:43 smithi181 ceph-mon[106466]: failed to encode map e434 with expected crc 2024-03-29T04:59:44.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:43 smithi181 ceph-mon[101970]: failed to encode map e434 with expected crc 2024-03-29T04:59:44.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:43 smithi181 ceph-mon[101970]: failed to encode map e434 with expected crc 2024-03-29T04:59:44.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:43 smithi181 ceph-mon[101970]: osdmap e434: 8 total, 8 up, 8 in 2024-03-29T04:59:44.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:43 smithi181 ceph-mon[101970]: failed to encode map e434 with expected crc 2024-03-29T04:59:44.783 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage 2024-03-29T04:59:45.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:44 smithi187 ceph-mon[93536]: failed to encode map e435 with expected crc 2024-03-29T04:59:45.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:44 smithi187 ceph-mon[93536]: failed to encode map e435 with expected crc 2024-03-29T04:59:45.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:44 smithi187 ceph-mon[93536]: osdmap e435: 8 total, 8 up, 8 in 2024-03-29T04:59:45.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:44 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:59:45.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:44 smithi187 ceph-mon[93536]: pgmap v312: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:59:45.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:44 smithi181 ceph-mon[101970]: failed to encode map e435 with expected crc 2024-03-29T04:59:45.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:44 smithi181 ceph-mon[101970]: failed to encode map e435 with expected crc 2024-03-29T04:59:45.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:44 smithi181 ceph-mon[101970]: osdmap e435: 8 total, 8 up, 8 in 2024-03-29T04:59:45.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:44 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:59:45.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:44 smithi181 ceph-mon[101970]: pgmap v312: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:59:45.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:44 smithi181 ceph-mon[106466]: failed to encode map e435 with expected crc 2024-03-29T04:59:45.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:44 smithi181 ceph-mon[106466]: failed to encode map e435 with expected crc 2024-03-29T04:59:45.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:44 smithi181 ceph-mon[106466]: osdmap e435: 8 total, 8 up, 8 in 2024-03-29T04:59:45.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:44 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:59:45.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:44 smithi181 ceph-mon[106466]: pgmap v312: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:59:46.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:45 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:59:46.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:45 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:59:46.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:45 smithi187 ceph-mon[93536]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:45 smithi187 ceph-mon[93536]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:45 smithi187 ceph-mon[93536]: osdmap e436: 8 total, 8 up, 8 in 2024-03-29T04:59:46.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:45 smithi187 ceph-mon[93536]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:45 smithi187 ceph-mon[93536]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:45 smithi187 ceph-mon[93536]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:45 smithi187 ceph-mon[93536]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:45 smithi187 ceph-mon[93536]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:45 smithi187 ceph-mon[93536]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:59:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:59:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[101970]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[101970]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[101970]: osdmap e436: 8 total, 8 up, 8 in 2024-03-29T04:59:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[101970]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[101970]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[101970]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[101970]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[101970]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[101970]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:59:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T04:59:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[106466]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[106466]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[106466]: osdmap e436: 8 total, 8 up, 8 in 2024-03-29T04:59:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[106466]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[106466]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[106466]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[106466]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[106466]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:45 smithi181 ceph-mon[106466]: failed to encode map e436 with expected crc 2024-03-29T04:59:46.909 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage (2126 ms) 2024-03-29T04:59:46.909 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool 2024-03-29T04:59:47.006 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool (97 ms) 2024-03-29T04:59:47.006 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled 2024-03-29T04:59:47.081 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled (75 ms) 2024-03-29T04:59:47.081 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage 2024-03-29T04:59:47.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:46 smithi187 ceph-mon[93536]: failed to encode map e437 with expected crc 2024-03-29T04:59:47.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:46 smithi187 ceph-mon[93536]: failed to encode map e437 with expected crc 2024-03-29T04:59:47.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:46 smithi187 ceph-mon[93536]: osdmap e437: 8 total, 8 up, 8 in 2024-03-29T04:59:47.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:46 smithi187 ceph-mon[93536]: failed to encode map e437 with expected crc 2024-03-29T04:59:47.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:46 smithi187 ceph-mon[93536]: pgmap v315: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:59:47.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:46 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:47.178 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage (97 ms) 2024-03-29T04:59:47.178 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap 2024-03-29T04:59:47.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:46 smithi181 ceph-mon[101970]: failed to encode map e437 with expected crc 2024-03-29T04:59:47.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:46 smithi181 ceph-mon[101970]: failed to encode map e437 with expected crc 2024-03-29T04:59:47.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:46 smithi181 ceph-mon[101970]: osdmap e437: 8 total, 8 up, 8 in 2024-03-29T04:59:47.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:46 smithi181 ceph-mon[101970]: failed to encode map e437 with expected crc 2024-03-29T04:59:47.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:46 smithi181 ceph-mon[101970]: pgmap v315: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:59:47.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:46 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:47.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:46 smithi181 ceph-mon[106466]: failed to encode map e437 with expected crc 2024-03-29T04:59:47.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:46 smithi181 ceph-mon[106466]: failed to encode map e437 with expected crc 2024-03-29T04:59:47.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:46 smithi181 ceph-mon[106466]: osdmap e437: 8 total, 8 up, 8 in 2024-03-29T04:59:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:46 smithi181 ceph-mon[106466]: failed to encode map e437 with expected crc 2024-03-29T04:59:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:46 smithi181 ceph-mon[106466]: pgmap v315: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-03-29T04:59:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:46 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:47.264 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap (86 ms) 2024-03-29T04:59:47.265 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModePool 2024-03-29T04:59:47.356 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModePool (91 ms) 2024-03-29T04:59:47.356 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled 2024-03-29T04:59:47.461 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled (105 ms) 2024-03-29T04:59:47.461 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.DisableImageMirrorWithPeer 2024-03-29T04:59:47.461 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:59:47.461 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.DisableImageMirrorWithPeer (0 ms) 2024-03-29T04:59:47.461 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.DisableJournalingWithPeer 2024-03-29T04:59:47.461 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T04:59:47.461 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.DisableJournalingWithPeer (0 ms) 2024-03-29T04:59:47.461 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling 2024-03-29T04:59:47.527 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling (66 ms) 2024-03-29T04:59:47.527 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling 2024-03-29T04:59:47.603 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling (76 ms) 2024-03-29T04:59:47.603 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling 2024-03-29T04:59:48.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:47 smithi187 ceph-mon[93536]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:59:48.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:47 smithi187 ceph-mon[93536]: failed to encode map e438 with expected crc 2024-03-29T04:59:48.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:47 smithi187 ceph-mon[93536]: failed to encode map e438 with expected crc 2024-03-29T04:59:48.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:47 smithi187 ceph-mon[93536]: osdmap e438: 8 total, 8 up, 8 in 2024-03-29T04:59:48.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:47 smithi187 ceph-mon[93536]: failed to encode map e438 with expected crc 2024-03-29T04:59:48.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:47 smithi187 ceph-mon[93536]: failed to encode map e438 with expected crc 2024-03-29T04:59:48.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:47 smithi187 ceph-mon[93536]: failed to encode map e438 with expected crc 2024-03-29T04:59:48.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:47 smithi181 ceph-mon[106466]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:59:48.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:47 smithi181 ceph-mon[106466]: failed to encode map e438 with expected crc 2024-03-29T04:59:48.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:47 smithi181 ceph-mon[106466]: failed to encode map e438 with expected crc 2024-03-29T04:59:48.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:47 smithi181 ceph-mon[106466]: osdmap e438: 8 total, 8 up, 8 in 2024-03-29T04:59:48.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:47 smithi181 ceph-mon[106466]: failed to encode map e438 with expected crc 2024-03-29T04:59:48.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:47 smithi181 ceph-mon[106466]: failed to encode map e438 with expected crc 2024-03-29T04:59:48.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:47 smithi181 ceph-mon[106466]: failed to encode map e438 with expected crc 2024-03-29T04:59:48.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:47 smithi181 ceph-mon[101970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:59:48.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:47 smithi181 ceph-mon[101970]: failed to encode map e438 with expected crc 2024-03-29T04:59:48.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:47 smithi181 ceph-mon[101970]: failed to encode map e438 with expected crc 2024-03-29T04:59:48.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:47 smithi181 ceph-mon[101970]: osdmap e438: 8 total, 8 up, 8 in 2024-03-29T04:59:48.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:47 smithi181 ceph-mon[101970]: failed to encode map e438 with expected crc 2024-03-29T04:59:48.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:47 smithi181 ceph-mon[101970]: failed to encode map e438 with expected crc 2024-03-29T04:59:48.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:47 smithi181 ceph-mon[101970]: failed to encode map e438 with expected crc 2024-03-29T04:59:48.859 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling (1256 ms) 2024-03-29T04:59:48.860 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock 2024-03-29T04:59:49.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:48 smithi187 ceph-mon[93536]: failed to encode map e439 with expected crc 2024-03-29T04:59:49.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:48 smithi187 ceph-mon[93536]: failed to encode map e439 with expected crc 2024-03-29T04:59:49.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:48 smithi187 ceph-mon[93536]: osdmap e439: 8 total, 8 up, 8 in 2024-03-29T04:59:49.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:48 smithi187 ceph-mon[93536]: failed to encode map e439 with expected crc 2024-03-29T04:59:49.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:48 smithi187 ceph-mon[93536]: failed to encode map e439 with expected crc 2024-03-29T04:59:49.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:48 smithi187 ceph-mon[93536]: pgmap v318: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 155 KiB/s rd, 21 KiB/s wr, 224 op/s 2024-03-29T04:59:49.203 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:59:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:59:48] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.43.0" 2024-03-29T04:59:49.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:48 smithi181 ceph-mon[101970]: failed to encode map e439 with expected crc 2024-03-29T04:59:49.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:48 smithi181 ceph-mon[101970]: failed to encode map e439 with expected crc 2024-03-29T04:59:49.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:48 smithi181 ceph-mon[101970]: osdmap e439: 8 total, 8 up, 8 in 2024-03-29T04:59:49.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:48 smithi181 ceph-mon[101970]: failed to encode map e439 with expected crc 2024-03-29T04:59:49.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:48 smithi181 ceph-mon[101970]: failed to encode map e439 with expected crc 2024-03-29T04:59:49.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:48 smithi181 ceph-mon[101970]: pgmap v318: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 155 KiB/s rd, 21 KiB/s wr, 224 op/s 2024-03-29T04:59:49.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:48 smithi181 ceph-mon[106466]: failed to encode map e439 with expected crc 2024-03-29T04:59:49.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:48 smithi181 ceph-mon[106466]: failed to encode map e439 with expected crc 2024-03-29T04:59:49.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:48 smithi181 ceph-mon[106466]: osdmap e439: 8 total, 8 up, 8 in 2024-03-29T04:59:49.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:48 smithi181 ceph-mon[106466]: failed to encode map e439 with expected crc 2024-03-29T04:59:49.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:48 smithi181 ceph-mon[106466]: failed to encode map e439 with expected crc 2024-03-29T04:59:49.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:48 smithi181 ceph-mon[106466]: pgmap v318: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 155 KiB/s rd, 21 KiB/s wr, 224 op/s 2024-03-29T04:59:50.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:49 smithi187 ceph-mon[93536]: failed to encode map e440 with expected crc 2024-03-29T04:59:50.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:49 smithi187 ceph-mon[93536]: failed to encode map e440 with expected crc 2024-03-29T04:59:50.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:49 smithi187 ceph-mon[93536]: osdmap e440: 8 total, 8 up, 8 in 2024-03-29T04:59:50.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:49 smithi187 ceph-mon[93536]: failed to encode map e440 with expected crc 2024-03-29T04:59:50.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:49 smithi181 ceph-mon[106466]: failed to encode map e440 with expected crc 2024-03-29T04:59:50.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:49 smithi181 ceph-mon[106466]: failed to encode map e440 with expected crc 2024-03-29T04:59:50.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:49 smithi181 ceph-mon[106466]: osdmap e440: 8 total, 8 up, 8 in 2024-03-29T04:59:50.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:49 smithi181 ceph-mon[106466]: failed to encode map e440 with expected crc 2024-03-29T04:59:50.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:49 smithi181 ceph-mon[101970]: failed to encode map e440 with expected crc 2024-03-29T04:59:50.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:49 smithi181 ceph-mon[101970]: failed to encode map e440 with expected crc 2024-03-29T04:59:50.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:49 smithi181 ceph-mon[101970]: osdmap e440: 8 total, 8 up, 8 in 2024-03-29T04:59:50.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:49 smithi181 ceph-mon[101970]: failed to encode map e440 with expected crc 2024-03-29T04:59:50.878 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock (2016 ms) 2024-03-29T04:59:50.878 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeDisabled 2024-03-29T04:59:50.968 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeDisabled (93 ms) 2024-03-29T04:59:50.969 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage 2024-03-29T04:59:51.054 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage (86 ms) 2024-03-29T04:59:51.055 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool 2024-03-29T04:59:51.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:50 smithi187 ceph-mon[93536]: failed to encode map e441 with expected crc 2024-03-29T04:59:51.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:50 smithi187 ceph-mon[93536]: failed to encode map e441 with expected crc 2024-03-29T04:59:51.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:50 smithi187 ceph-mon[93536]: osdmap e441: 8 total, 8 up, 8 in 2024-03-29T04:59:51.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:50 smithi187 ceph-mon[93536]: failed to encode map e441 with expected crc 2024-03-29T04:59:51.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:50 smithi187 ceph-mon[93536]: failed to encode map e441 with expected crc 2024-03-29T04:59:51.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:50 smithi187 ceph-mon[93536]: pgmap v321: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 155 KiB/s rd, 21 KiB/s wr, 224 op/s 2024-03-29T04:59:51.145 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool (91 ms) 2024-03-29T04:59:51.146 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling 2024-03-29T04:59:51.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:50 smithi181 ceph-mon[101970]: failed to encode map e441 with expected crc 2024-03-29T04:59:51.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:50 smithi181 ceph-mon[101970]: failed to encode map e441 with expected crc 2024-03-29T04:59:51.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:50 smithi181 ceph-mon[101970]: osdmap e441: 8 total, 8 up, 8 in 2024-03-29T04:59:51.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:50 smithi181 ceph-mon[101970]: failed to encode map e441 with expected crc 2024-03-29T04:59:51.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:50 smithi181 ceph-mon[101970]: failed to encode map e441 with expected crc 2024-03-29T04:59:51.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:50 smithi181 ceph-mon[101970]: pgmap v321: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 155 KiB/s rd, 21 KiB/s wr, 224 op/s 2024-03-29T04:59:51.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:50 smithi181 ceph-mon[106466]: failed to encode map e441 with expected crc 2024-03-29T04:59:51.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:50 smithi181 ceph-mon[106466]: failed to encode map e441 with expected crc 2024-03-29T04:59:51.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:50 smithi181 ceph-mon[106466]: osdmap e441: 8 total, 8 up, 8 in 2024-03-29T04:59:51.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:50 smithi181 ceph-mon[106466]: failed to encode map e441 with expected crc 2024-03-29T04:59:51.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:50 smithi181 ceph-mon[106466]: failed to encode map e441 with expected crc 2024-03-29T04:59:51.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:50 smithi181 ceph-mon[106466]: pgmap v321: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 155 KiB/s rd, 21 KiB/s wr, 224 op/s 2024-03-29T04:59:51.221 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling (77 ms) 2024-03-29T04:59:51.222 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling 2024-03-29T04:59:51.294 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling (72 ms) 2024-03-29T04:59:51.294 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeDisabled 2024-03-29T04:59:51.376 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeDisabled (82 ms) 2024-03-29T04:59:51.376 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage 2024-03-29T04:59:51.468 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage (92 ms) 2024-03-29T04:59:51.468 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled 2024-03-29T04:59:51.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:51 smithi181 ceph-mon[106466]: failed to encode map e442 with expected crc 2024-03-29T04:59:51.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:51 smithi181 ceph-mon[106466]: failed to encode map e442 with expected crc 2024-03-29T04:59:51.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:51 smithi181 ceph-mon[106466]: osdmap e442: 8 total, 8 up, 8 in 2024-03-29T04:59:51.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:51 smithi181 ceph-mon[106466]: failed to encode map e442 with expected crc 2024-03-29T04:59:51.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:51 smithi181 ceph-mon[106466]: failed to encode map e442 with expected crc 2024-03-29T04:59:51.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:51 smithi181 ceph-mon[101970]: failed to encode map e442 with expected crc 2024-03-29T04:59:51.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:51 smithi181 ceph-mon[101970]: failed to encode map e442 with expected crc 2024-03-29T04:59:51.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:51 smithi181 ceph-mon[101970]: osdmap e442: 8 total, 8 up, 8 in 2024-03-29T04:59:51.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:51 smithi181 ceph-mon[101970]: failed to encode map e442 with expected crc 2024-03-29T04:59:51.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:51 smithi181 ceph-mon[101970]: failed to encode map e442 with expected crc 2024-03-29T04:59:52.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:51 smithi187 ceph-mon[93536]: failed to encode map e442 with expected crc 2024-03-29T04:59:52.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:51 smithi187 ceph-mon[93536]: failed to encode map e442 with expected crc 2024-03-29T04:59:52.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:51 smithi187 ceph-mon[93536]: osdmap e442: 8 total, 8 up, 8 in 2024-03-29T04:59:52.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:51 smithi187 ceph-mon[93536]: failed to encode map e442 with expected crc 2024-03-29T04:59:52.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:51 smithi187 ceph-mon[93536]: failed to encode map e442 with expected crc 2024-03-29T04:59:52.902 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled (1433 ms) 2024-03-29T04:59:52.902 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModePool 2024-03-29T04:59:53.017 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModePool (116 ms) 2024-03-29T04:59:53.017 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModePool 2024-03-29T04:59:53.114 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModePool (97 ms) 2024-03-29T04:59:53.114 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModeImage 2024-03-29T04:59:53.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:52 smithi187 ceph-mon[93536]: failed to encode map e443 with expected crc 2024-03-29T04:59:53.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:52 smithi187 ceph-mon[93536]: osdmap e443: 8 total, 8 up, 8 in 2024-03-29T04:59:53.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:52 smithi187 ceph-mon[93536]: failed to encode map e443 with expected crc 2024-03-29T04:59:53.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:52 smithi187 ceph-mon[93536]: failed to encode map e443 with expected crc 2024-03-29T04:59:53.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:52 smithi187 ceph-mon[93536]: failed to encode map e443 with expected crc 2024-03-29T04:59:53.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:52 smithi187 ceph-mon[93536]: failed to encode map e443 with expected crc 2024-03-29T04:59:53.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:52 smithi187 ceph-mon[93536]: pgmap v324: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s rd, 13 KiB/s wr, 110 op/s 2024-03-29T04:59:53.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:52 smithi181 ceph-mon[101970]: failed to encode map e443 with expected crc 2024-03-29T04:59:53.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:52 smithi181 ceph-mon[101970]: osdmap e443: 8 total, 8 up, 8 in 2024-03-29T04:59:53.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:52 smithi181 ceph-mon[101970]: failed to encode map e443 with expected crc 2024-03-29T04:59:53.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:52 smithi181 ceph-mon[101970]: failed to encode map e443 with expected crc 2024-03-29T04:59:53.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:52 smithi181 ceph-mon[101970]: failed to encode map e443 with expected crc 2024-03-29T04:59:53.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:52 smithi181 ceph-mon[101970]: failed to encode map e443 with expected crc 2024-03-29T04:59:53.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:52 smithi181 ceph-mon[101970]: pgmap v324: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s rd, 13 KiB/s wr, 110 op/s 2024-03-29T04:59:53.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:52 smithi181 ceph-mon[106466]: failed to encode map e443 with expected crc 2024-03-29T04:59:53.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:52 smithi181 ceph-mon[106466]: osdmap e443: 8 total, 8 up, 8 in 2024-03-29T04:59:53.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:52 smithi181 ceph-mon[106466]: failed to encode map e443 with expected crc 2024-03-29T04:59:53.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:52 smithi181 ceph-mon[106466]: failed to encode map e443 with expected crc 2024-03-29T04:59:53.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:52 smithi181 ceph-mon[106466]: failed to encode map e443 with expected crc 2024-03-29T04:59:53.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:52 smithi181 ceph-mon[106466]: failed to encode map e443 with expected crc 2024-03-29T04:59:53.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:52 smithi181 ceph-mon[106466]: pgmap v324: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s rd, 13 KiB/s wr, 110 op/s 2024-03-29T04:59:53.206 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModeImage (92 ms) 2024-03-29T04:59:53.206 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode 2024-03-29T04:59:53.348 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode (142 ms) 2024-03-29T04:59:53.348 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode 2024-03-29T04:59:53.491 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode (142 ms) 2024-03-29T04:59:53.491 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode 2024-03-29T04:59:53.619 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode (129 ms) 2024-03-29T04:59:53.620 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode 2024-03-29T04:59:53.755 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode (136 ms) 2024-03-29T04:59:53.755 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode 2024-03-29T04:59:53.902 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode (147 ms) 2024-03-29T04:59:53.902 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode 2024-03-29T04:59:54.065 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode (163 ms) 2024-03-29T04:59:54.065 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageEnabled 2024-03-29T04:59:54.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:53 smithi187 ceph-mon[93536]: failed to encode map e444 with expected crc 2024-03-29T04:59:54.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:53 smithi187 ceph-mon[93536]: failed to encode map e444 with expected crc 2024-03-29T04:59:54.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:53 smithi187 ceph-mon[93536]: osdmap e444: 8 total, 8 up, 8 in 2024-03-29T04:59:54.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:53 smithi187 ceph-mon[93536]: failed to encode map e444 with expected crc 2024-03-29T04:59:54.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:53 smithi187 ceph-mon[93536]: failed to encode map e444 with expected crc 2024-03-29T04:59:54.144 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageEnabled (79 ms) 2024-03-29T04:59:54.144 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDisabled 2024-03-29T04:59:54.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:53 smithi181 ceph-mon[106466]: failed to encode map e444 with expected crc 2024-03-29T04:59:54.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:53 smithi181 ceph-mon[106466]: failed to encode map e444 with expected crc 2024-03-29T04:59:54.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:53 smithi181 ceph-mon[106466]: osdmap e444: 8 total, 8 up, 8 in 2024-03-29T04:59:54.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:53 smithi181 ceph-mon[106466]: failed to encode map e444 with expected crc 2024-03-29T04:59:54.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:53 smithi181 ceph-mon[106466]: failed to encode map e444 with expected crc 2024-03-29T04:59:54.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:53 smithi181 ceph-mon[101970]: failed to encode map e444 with expected crc 2024-03-29T04:59:54.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:53 smithi181 ceph-mon[101970]: failed to encode map e444 with expected crc 2024-03-29T04:59:54.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:53 smithi181 ceph-mon[101970]: osdmap e444: 8 total, 8 up, 8 in 2024-03-29T04:59:54.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:53 smithi181 ceph-mon[101970]: failed to encode map e444 with expected crc 2024-03-29T04:59:54.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:53 smithi181 ceph-mon[101970]: failed to encode map e444 with expected crc 2024-03-29T04:59:54.214 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDisabled (70 ms) 2024-03-29T04:59:54.214 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.RemoveImage_With_ImageWithoutJournal 2024-03-29T04:59:54.277 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.RemoveImage_With_ImageWithoutJournal (63 ms) 2024-03-29T04:59:54.277 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDemoted 2024-03-29T04:59:54.387 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDemoted (110 ms) 2024-03-29T04:59:54.387 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.TrashMoveRestore_PoolMode 2024-03-29T04:59:54.500 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.TrashMoveRestore_PoolMode (113 ms) 2024-03-29T04:59:54.500 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled 2024-03-29T04:59:54.596 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled (96 ms) 2024-03-29T04:59:54.596 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled 2024-03-29T04:59:54.697 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled (101 ms) 2024-03-29T04:59:54.698 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.MirrorStatusList 2024-03-29T04:59:54.765 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.MirrorStatusList (68 ms) 2024-03-29T04:59:54.765 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.RemoveBootstrapped 2024-03-29T04:59:54.856 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.RemoveBootstrapped (91 ms) 2024-03-29T04:59:54.856 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.AioPromoteDemote 2024-03-29T04:59:55.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:54 smithi187 ceph-mon[93536]: pgmap v326: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 567 KiB/s rd, 72 KiB/s wr, 824 op/s 2024-03-29T04:59:55.191 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.AioPromoteDemote (335 ms) 2024-03-29T04:59:55.191 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.AioGetInfo 2024-03-29T04:59:55.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:54 smithi181 ceph-mon[101970]: pgmap v326: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 567 KiB/s rd, 72 KiB/s wr, 824 op/s 2024-03-29T04:59:55.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:54 smithi181 ceph-mon[106466]: pgmap v326: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 567 KiB/s rd, 72 KiB/s wr, 824 op/s 2024-03-29T04:59:55.517 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.AioGetInfo (326 ms) 2024-03-29T04:59:55.517 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.AioGetStatus 2024-03-29T04:59:55.725 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.AioGetStatus (208 ms) 2024-03-29T04:59:55.725 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.SiteName 2024-03-29T04:59:55.732 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.SiteName (7 ms) 2024-03-29T04:59:55.732 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.Bootstrap 2024-03-29T04:59:56.510 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.Bootstrap (778 ms) 2024-03-29T04:59:56.511 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.PeerDirection 2024-03-29T04:59:56.518 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.PeerDirection (8 ms) 2024-03-29T04:59:56.518 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.Snapshot 2024-03-29T04:59:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:56 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' 2024-03-29T04:59:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:56 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-29T04:59:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:56 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-29T04:59:57.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:56 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-03-29T04:59:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:56 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-29T04:59:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:56 smithi187 ceph-mon[93536]: pgmap v327: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 384 KiB/s rd, 49 KiB/s wr, 557 op/s 2024-03-29T04:59:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:59:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:56 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-03-29T04:59:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:56 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' 2024-03-29T04:59:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:56 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-03-29T04:59:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:56 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-03-29T04:59:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:56 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-29T04:59:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:56 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/d6db801b-4971-4970-a193-1fc8be1e574d"}]: dispatch 2024-03-29T04:59:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:56 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/d6db801b-4971-4970-a193-1fc8be1e574d"}]': finished 2024-03-29T04:59:57.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' 2024-03-29T04:59:57.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-29T04:59:57.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-29T04:59:57.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-03-29T04:59:57.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-29T04:59:57.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[106466]: pgmap v327: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 384 KiB/s rd, 49 KiB/s wr, 557 op/s 2024-03-29T04:59:57.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:59:57.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:57.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-03-29T04:59:57.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' 2024-03-29T04:59:57.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-03-29T04:59:57.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-03-29T04:59:57.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-29T04:59:57.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/d6db801b-4971-4970-a193-1fc8be1e574d"}]: dispatch 2024-03-29T04:59:57.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/d6db801b-4971-4970-a193-1fc8be1e574d"}]': finished 2024-03-29T04:59:57.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' 2024-03-29T04:59:57.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-29T04:59:57.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-29T04:59:57.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-03-29T04:59:57.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-29T04:59:57.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[101970]: pgmap v327: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 384 KiB/s rd, 49 KiB/s wr, 557 op/s 2024-03-29T04:59:57.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:59:57.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T04:59:57.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-03-29T04:59:57.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' 2024-03-29T04:59:57.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-03-29T04:59:57.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-03-29T04:59:57.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-29T04:59:57.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/d6db801b-4971-4970-a193-1fc8be1e574d"}]: dispatch 2024-03-29T04:59:57.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:56 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/d6db801b-4971-4970-a193-1fc8be1e574d"}]': finished 2024-03-29T04:59:58.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:58 smithi187 ceph-mon[93536]: failed to encode map e445 with expected crc 2024-03-29T04:59:58.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:58 smithi187 ceph-mon[93536]: failed to encode map e445 with expected crc 2024-03-29T04:59:58.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:58 smithi187 ceph-mon[93536]: osdmap e445: 8 total, 8 up, 8 in 2024-03-29T04:59:58.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:58 smithi187 ceph-mon[93536]: failed to encode map e445 with expected crc 2024-03-29T04:59:58.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:58 smithi187 ceph-mon[93536]: pgmap v329: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 245 KiB/s wr, 2.37k op/s 2024-03-29T04:59:58.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:58 smithi181 ceph-mon[106466]: failed to encode map e445 with expected crc 2024-03-29T04:59:58.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:58 smithi181 ceph-mon[106466]: failed to encode map e445 with expected crc 2024-03-29T04:59:58.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:58 smithi181 ceph-mon[106466]: osdmap e445: 8 total, 8 up, 8 in 2024-03-29T04:59:58.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:58 smithi181 ceph-mon[106466]: failed to encode map e445 with expected crc 2024-03-29T04:59:58.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:58 smithi181 ceph-mon[106466]: pgmap v329: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 245 KiB/s wr, 2.37k op/s 2024-03-29T04:59:58.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:58 smithi181 ceph-mon[101970]: failed to encode map e445 with expected crc 2024-03-29T04:59:58.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:58 smithi181 ceph-mon[101970]: failed to encode map e445 with expected crc 2024-03-29T04:59:58.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:58 smithi181 ceph-mon[101970]: osdmap e445: 8 total, 8 up, 8 in 2024-03-29T04:59:58.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:58 smithi181 ceph-mon[101970]: failed to encode map e445 with expected crc 2024-03-29T04:59:58.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:58 smithi181 ceph-mon[101970]: pgmap v329: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 245 KiB/s wr, 2.37k op/s 2024-03-29T04:59:59.203 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 04:59:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:04:59:58] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.43.0" 2024-03-29T04:59:59.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:59 smithi187 ceph-mon[93536]: failed to encode map e446 with expected crc 2024-03-29T04:59:59.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:59 smithi187 ceph-mon[93536]: failed to encode map e446 with expected crc 2024-03-29T04:59:59.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:59 smithi187 ceph-mon[93536]: osdmap e446: 8 total, 8 up, 8 in 2024-03-29T04:59:59.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 04:59:59 smithi187 ceph-mon[93536]: failed to encode map e446 with expected crc 2024-03-29T04:59:59.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:59 smithi181 ceph-mon[101970]: failed to encode map e446 with expected crc 2024-03-29T04:59:59.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:59 smithi181 ceph-mon[101970]: failed to encode map e446 with expected crc 2024-03-29T04:59:59.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:59 smithi181 ceph-mon[101970]: osdmap e446: 8 total, 8 up, 8 in 2024-03-29T04:59:59.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 04:59:59 smithi181 ceph-mon[101970]: failed to encode map e446 with expected crc 2024-03-29T04:59:59.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:59 smithi181 ceph-mon[106466]: failed to encode map e446 with expected crc 2024-03-29T04:59:59.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:59 smithi181 ceph-mon[106466]: failed to encode map e446 with expected crc 2024-03-29T04:59:59.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:59 smithi181 ceph-mon[106466]: osdmap e446: 8 total, 8 up, 8 in 2024-03-29T04:59:59.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 04:59:59 smithi181 ceph-mon[106466]: failed to encode map e446 with expected crc 2024-03-29T05:00:00.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:00 smithi187 ceph-mon[93536]: failed to encode map e447 with expected crc 2024-03-29T05:00:00.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:00 smithi187 ceph-mon[93536]: failed to encode map e447 with expected crc 2024-03-29T05:00:00.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:00 smithi187 ceph-mon[93536]: osdmap e447: 8 total, 8 up, 8 in 2024-03-29T05:00:00.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:00 smithi187 ceph-mon[93536]: failed to encode map e447 with expected crc 2024-03-29T05:00:00.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:00 smithi187 ceph-mon[93536]: pgmap v332: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 207 KiB/s wr, 1.91k op/s 2024-03-29T05:00:00.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:00 smithi187 ceph-mon[93536]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T05:00:00.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:00 smithi187 ceph-mon[93536]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T05:00:00.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:00 smithi187 ceph-mon[93536]: daemon grafana.a on smithi187 is in error state 2024-03-29T05:00:00.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:00 smithi181 ceph-mon[106466]: failed to encode map e447 with expected crc 2024-03-29T05:00:00.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:00 smithi181 ceph-mon[106466]: failed to encode map e447 with expected crc 2024-03-29T05:00:00.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:00 smithi181 ceph-mon[106466]: osdmap e447: 8 total, 8 up, 8 in 2024-03-29T05:00:00.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:00 smithi181 ceph-mon[106466]: failed to encode map e447 with expected crc 2024-03-29T05:00:00.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:00 smithi181 ceph-mon[106466]: pgmap v332: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 207 KiB/s wr, 1.91k op/s 2024-03-29T05:00:00.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:00 smithi181 ceph-mon[106466]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T05:00:00.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:00 smithi181 ceph-mon[106466]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T05:00:00.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:00 smithi181 ceph-mon[106466]: daemon grafana.a on smithi187 is in error state 2024-03-29T05:00:00.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:00 smithi181 ceph-mon[101970]: failed to encode map e447 with expected crc 2024-03-29T05:00:00.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:00 smithi181 ceph-mon[101970]: failed to encode map e447 with expected crc 2024-03-29T05:00:00.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:00 smithi181 ceph-mon[101970]: osdmap e447: 8 total, 8 up, 8 in 2024-03-29T05:00:00.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:00 smithi181 ceph-mon[101970]: failed to encode map e447 with expected crc 2024-03-29T05:00:00.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:00 smithi181 ceph-mon[101970]: pgmap v332: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 207 KiB/s wr, 1.91k op/s 2024-03-29T05:00:00.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:00 smithi181 ceph-mon[101970]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T05:00:00.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:00 smithi181 ceph-mon[101970]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T05:00:00.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:00 smithi181 ceph-mon[101970]: daemon grafana.a on smithi187 is in error state 2024-03-29T05:00:01.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:01 smithi187 ceph-mon[93536]: failed to encode map e447 with expected crc 2024-03-29T05:00:01.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:01 smithi187 ceph-mon[93536]: failed to encode map e448 with expected crc 2024-03-29T05:00:01.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:01 smithi187 ceph-mon[93536]: failed to encode map e448 with expected crc 2024-03-29T05:00:01.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:01 smithi187 ceph-mon[93536]: osdmap e448: 8 total, 8 up, 8 in 2024-03-29T05:00:01.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:01 smithi187 ceph-mon[93536]: failed to encode map e448 with expected crc 2024-03-29T05:00:01.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:01 smithi181 ceph-mon[101970]: failed to encode map e447 with expected crc 2024-03-29T05:00:01.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:01 smithi181 ceph-mon[101970]: failed to encode map e448 with expected crc 2024-03-29T05:00:01.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:01 smithi181 ceph-mon[101970]: failed to encode map e448 with expected crc 2024-03-29T05:00:01.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:01 smithi181 ceph-mon[101970]: osdmap e448: 8 total, 8 up, 8 in 2024-03-29T05:00:01.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:01 smithi181 ceph-mon[101970]: failed to encode map e448 with expected crc 2024-03-29T05:00:01.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:01 smithi181 ceph-mon[106466]: failed to encode map e447 with expected crc 2024-03-29T05:00:01.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:01 smithi181 ceph-mon[106466]: failed to encode map e448 with expected crc 2024-03-29T05:00:01.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:01 smithi181 ceph-mon[106466]: failed to encode map e448 with expected crc 2024-03-29T05:00:01.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:01 smithi181 ceph-mon[106466]: osdmap e448: 8 total, 8 up, 8 in 2024-03-29T05:00:01.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:01 smithi181 ceph-mon[106466]: failed to encode map e448 with expected crc 2024-03-29T05:00:02.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:02 smithi187 ceph-mon[93536]: failed to encode map e449 with expected crc 2024-03-29T05:00:02.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:02 smithi187 ceph-mon[93536]: failed to encode map e449 with expected crc 2024-03-29T05:00:02.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:02 smithi187 ceph-mon[93536]: osdmap e449: 8 total, 8 up, 8 in 2024-03-29T05:00:02.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:02 smithi187 ceph-mon[93536]: failed to encode map e449 with expected crc 2024-03-29T05:00:02.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:02 smithi187 ceph-mon[93536]: failed to encode map e449 with expected crc 2024-03-29T05:00:02.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:02 smithi187 ceph-mon[93536]: pgmap v335: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 511 B/s wr, 12 op/s 2024-03-29T05:00:02.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:02 smithi181 ceph-mon[106466]: failed to encode map e449 with expected crc 2024-03-29T05:00:02.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:02 smithi181 ceph-mon[106466]: failed to encode map e449 with expected crc 2024-03-29T05:00:02.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:02 smithi181 ceph-mon[106466]: osdmap e449: 8 total, 8 up, 8 in 2024-03-29T05:00:02.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:02 smithi181 ceph-mon[106466]: failed to encode map e449 with expected crc 2024-03-29T05:00:02.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:02 smithi181 ceph-mon[106466]: failed to encode map e449 with expected crc 2024-03-29T05:00:02.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:02 smithi181 ceph-mon[106466]: pgmap v335: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 511 B/s wr, 12 op/s 2024-03-29T05:00:02.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:02 smithi181 ceph-mon[101970]: failed to encode map e449 with expected crc 2024-03-29T05:00:02.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:02 smithi181 ceph-mon[101970]: failed to encode map e449 with expected crc 2024-03-29T05:00:02.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:02 smithi181 ceph-mon[101970]: osdmap e449: 8 total, 8 up, 8 in 2024-03-29T05:00:02.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:02 smithi181 ceph-mon[101970]: failed to encode map e449 with expected crc 2024-03-29T05:00:02.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:02 smithi181 ceph-mon[101970]: failed to encode map e449 with expected crc 2024-03-29T05:00:02.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:02 smithi181 ceph-mon[101970]: pgmap v335: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 511 B/s wr, 12 op/s 2024-03-29T05:00:03.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:03 smithi187 ceph-mon[93536]: failed to encode map e450 with expected crc 2024-03-29T05:00:03.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:03 smithi187 ceph-mon[93536]: osdmap e450: 8 total, 8 up, 8 in 2024-03-29T05:00:03.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:03 smithi187 ceph-mon[93536]: failed to encode map e450 with expected crc 2024-03-29T05:00:03.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:03 smithi187 ceph-mon[93536]: failed to encode map e450 with expected crc 2024-03-29T05:00:03.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:03 smithi187 ceph-mon[93536]: failed to encode map e450 with expected crc 2024-03-29T05:00:03.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:03 smithi181 ceph-mon[106466]: failed to encode map e450 with expected crc 2024-03-29T05:00:03.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:03 smithi181 ceph-mon[106466]: osdmap e450: 8 total, 8 up, 8 in 2024-03-29T05:00:03.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:03 smithi181 ceph-mon[106466]: failed to encode map e450 with expected crc 2024-03-29T05:00:03.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:03 smithi181 ceph-mon[106466]: failed to encode map e450 with expected crc 2024-03-29T05:00:03.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:03 smithi181 ceph-mon[106466]: failed to encode map e450 with expected crc 2024-03-29T05:00:03.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:03 smithi181 ceph-mon[101970]: failed to encode map e450 with expected crc 2024-03-29T05:00:03.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:03 smithi181 ceph-mon[101970]: osdmap e450: 8 total, 8 up, 8 in 2024-03-29T05:00:03.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:03 smithi181 ceph-mon[101970]: failed to encode map e450 with expected crc 2024-03-29T05:00:03.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:03 smithi181 ceph-mon[101970]: failed to encode map e450 with expected crc 2024-03-29T05:00:03.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:03 smithi181 ceph-mon[101970]: failed to encode map e450 with expected crc 2024-03-29T05:00:04.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:04 smithi187 ceph-mon[93536]: failed to encode map e451 with expected crc 2024-03-29T05:00:04.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:04 smithi187 ceph-mon[93536]: failed to encode map e451 with expected crc 2024-03-29T05:00:04.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:04 smithi187 ceph-mon[93536]: osdmap e451: 8 total, 8 up, 8 in 2024-03-29T05:00:04.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:04 smithi187 ceph-mon[93536]: failed to encode map e451 with expected crc 2024-03-29T05:00:04.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:04 smithi187 ceph-mon[93536]: pgmap v338: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 7.0 KiB/s wr, 78 op/s 2024-03-29T05:00:04.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:04 smithi181 ceph-mon[106466]: failed to encode map e451 with expected crc 2024-03-29T05:00:04.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:04 smithi181 ceph-mon[106466]: failed to encode map e451 with expected crc 2024-03-29T05:00:04.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:04 smithi181 ceph-mon[106466]: osdmap e451: 8 total, 8 up, 8 in 2024-03-29T05:00:04.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:04 smithi181 ceph-mon[106466]: failed to encode map e451 with expected crc 2024-03-29T05:00:04.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:04 smithi181 ceph-mon[106466]: pgmap v338: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 7.0 KiB/s wr, 78 op/s 2024-03-29T05:00:04.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:04 smithi181 ceph-mon[101970]: failed to encode map e451 with expected crc 2024-03-29T05:00:04.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:04 smithi181 ceph-mon[101970]: failed to encode map e451 with expected crc 2024-03-29T05:00:04.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:04 smithi181 ceph-mon[101970]: osdmap e451: 8 total, 8 up, 8 in 2024-03-29T05:00:04.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:04 smithi181 ceph-mon[101970]: failed to encode map e451 with expected crc 2024-03-29T05:00:04.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:04 smithi181 ceph-mon[101970]: pgmap v338: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 7.0 KiB/s wr, 78 op/s 2024-03-29T05:00:05.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:05 smithi187 ceph-mon[93536]: failed to encode map e452 with expected crc 2024-03-29T05:00:05.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:05 smithi187 ceph-mon[93536]: failed to encode map e452 with expected crc 2024-03-29T05:00:05.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:05 smithi187 ceph-mon[93536]: osdmap e452: 8 total, 8 up, 8 in 2024-03-29T05:00:05.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:05 smithi187 ceph-mon[93536]: failed to encode map e452 with expected crc 2024-03-29T05:00:05.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:05 smithi187 ceph-mon[93536]: failed to encode map e452 with expected crc 2024-03-29T05:00:05.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:05 smithi187 ceph-mon[93536]: failed to encode map e452 with expected crc 2024-03-29T05:00:05.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:05 smithi181 ceph-mon[106466]: failed to encode map e452 with expected crc 2024-03-29T05:00:05.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:05 smithi181 ceph-mon[106466]: failed to encode map e452 with expected crc 2024-03-29T05:00:05.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:05 smithi181 ceph-mon[106466]: osdmap e452: 8 total, 8 up, 8 in 2024-03-29T05:00:05.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:05 smithi181 ceph-mon[106466]: failed to encode map e452 with expected crc 2024-03-29T05:00:05.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:05 smithi181 ceph-mon[106466]: failed to encode map e452 with expected crc 2024-03-29T05:00:05.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:05 smithi181 ceph-mon[106466]: failed to encode map e452 with expected crc 2024-03-29T05:00:05.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:05 smithi181 ceph-mon[101970]: failed to encode map e452 with expected crc 2024-03-29T05:00:05.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:05 smithi181 ceph-mon[101970]: failed to encode map e452 with expected crc 2024-03-29T05:00:05.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:05 smithi181 ceph-mon[101970]: osdmap e452: 8 total, 8 up, 8 in 2024-03-29T05:00:05.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:05 smithi181 ceph-mon[101970]: failed to encode map e452 with expected crc 2024-03-29T05:00:05.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:05 smithi181 ceph-mon[101970]: failed to encode map e452 with expected crc 2024-03-29T05:00:05.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:05 smithi181 ceph-mon[101970]: failed to encode map e452 with expected crc 2024-03-29T05:00:06.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:06 smithi187 ceph-mon[93536]: failed to encode map e453 with expected crc 2024-03-29T05:00:06.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:06 smithi187 ceph-mon[93536]: failed to encode map e453 with expected crc 2024-03-29T05:00:06.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:06 smithi187 ceph-mon[93536]: osdmap e453: 8 total, 8 up, 8 in 2024-03-29T05:00:06.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:06 smithi187 ceph-mon[93536]: failed to encode map e453 with expected crc 2024-03-29T05:00:06.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:06 smithi187 ceph-mon[93536]: failed to encode map e453 with expected crc 2024-03-29T05:00:06.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:06 smithi187 ceph-mon[93536]: pgmap v341: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 6.5 KiB/s wr, 65 op/s 2024-03-29T05:00:06.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:06 smithi181 ceph-mon[101970]: failed to encode map e453 with expected crc 2024-03-29T05:00:06.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:06 smithi181 ceph-mon[101970]: failed to encode map e453 with expected crc 2024-03-29T05:00:06.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:06 smithi181 ceph-mon[101970]: osdmap e453: 8 total, 8 up, 8 in 2024-03-29T05:00:06.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:06 smithi181 ceph-mon[101970]: failed to encode map e453 with expected crc 2024-03-29T05:00:06.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:06 smithi181 ceph-mon[101970]: failed to encode map e453 with expected crc 2024-03-29T05:00:06.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:06 smithi181 ceph-mon[101970]: pgmap v341: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 6.5 KiB/s wr, 65 op/s 2024-03-29T05:00:06.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:06 smithi181 ceph-mon[106466]: failed to encode map e453 with expected crc 2024-03-29T05:00:06.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:06 smithi181 ceph-mon[106466]: failed to encode map e453 with expected crc 2024-03-29T05:00:06.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:06 smithi181 ceph-mon[106466]: osdmap e453: 8 total, 8 up, 8 in 2024-03-29T05:00:06.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:06 smithi181 ceph-mon[106466]: failed to encode map e453 with expected crc 2024-03-29T05:00:06.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:06 smithi181 ceph-mon[106466]: failed to encode map e453 with expected crc 2024-03-29T05:00:06.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:06 smithi181 ceph-mon[106466]: pgmap v341: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 6.5 KiB/s wr, 65 op/s 2024-03-29T05:00:07.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:07 smithi187 ceph-mon[93536]: failed to encode map e454 with expected crc 2024-03-29T05:00:07.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:07 smithi187 ceph-mon[93536]: failed to encode map e454 with expected crc 2024-03-29T05:00:07.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:07 smithi187 ceph-mon[93536]: osdmap e454: 8 total, 8 up, 8 in 2024-03-29T05:00:07.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:07 smithi187 ceph-mon[93536]: failed to encode map e454 with expected crc 2024-03-29T05:00:07.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:07 smithi187 ceph-mon[93536]: failed to encode map e454 with expected crc 2024-03-29T05:00:07.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:07 smithi187 ceph-mon[93536]: osdmap e455: 8 total, 8 up, 8 in 2024-03-29T05:00:07.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:07 smithi181 ceph-mon[106466]: failed to encode map e454 with expected crc 2024-03-29T05:00:07.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:07 smithi181 ceph-mon[106466]: failed to encode map e454 with expected crc 2024-03-29T05:00:07.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:07 smithi181 ceph-mon[106466]: osdmap e454: 8 total, 8 up, 8 in 2024-03-29T05:00:07.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:07 smithi181 ceph-mon[106466]: failed to encode map e454 with expected crc 2024-03-29T05:00:07.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:07 smithi181 ceph-mon[106466]: failed to encode map e454 with expected crc 2024-03-29T05:00:07.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:07 smithi181 ceph-mon[106466]: osdmap e455: 8 total, 8 up, 8 in 2024-03-29T05:00:07.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:07 smithi181 ceph-mon[101970]: failed to encode map e454 with expected crc 2024-03-29T05:00:07.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:07 smithi181 ceph-mon[101970]: failed to encode map e454 with expected crc 2024-03-29T05:00:07.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:07 smithi181 ceph-mon[101970]: osdmap e454: 8 total, 8 up, 8 in 2024-03-29T05:00:07.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:07 smithi181 ceph-mon[101970]: failed to encode map e454 with expected crc 2024-03-29T05:00:07.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:07 smithi181 ceph-mon[101970]: failed to encode map e454 with expected crc 2024-03-29T05:00:07.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:07 smithi181 ceph-mon[101970]: osdmap e455: 8 total, 8 up, 8 in 2024-03-29T05:00:08.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:08 smithi187 ceph-mon[93536]: failed to encode map e455 with expected crc 2024-03-29T05:00:08.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:08 smithi187 ceph-mon[93536]: failed to encode map e455 with expected crc 2024-03-29T05:00:08.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:08 smithi187 ceph-mon[93536]: failed to encode map e455 with expected crc 2024-03-29T05:00:08.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:08 smithi187 ceph-mon[93536]: failed to encode map e455 with expected crc 2024-03-29T05:00:08.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:08 smithi187 ceph-mon[93536]: pgmap v344: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 6.5 KiB/s wr, 94 op/s 2024-03-29T05:00:08.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:08 smithi187 ceph-mon[93536]: osdmap e456: 8 total, 8 up, 8 in 2024-03-29T05:00:08.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:08 smithi181 ceph-mon[101970]: failed to encode map e455 with expected crc 2024-03-29T05:00:08.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:08 smithi181 ceph-mon[101970]: failed to encode map e455 with expected crc 2024-03-29T05:00:08.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:08 smithi181 ceph-mon[101970]: failed to encode map e455 with expected crc 2024-03-29T05:00:08.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:08 smithi181 ceph-mon[101970]: failed to encode map e455 with expected crc 2024-03-29T05:00:08.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:08 smithi181 ceph-mon[101970]: pgmap v344: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 6.5 KiB/s wr, 94 op/s 2024-03-29T05:00:08.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:08 smithi181 ceph-mon[101970]: osdmap e456: 8 total, 8 up, 8 in 2024-03-29T05:00:08.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:08 smithi181 ceph-mon[106466]: failed to encode map e455 with expected crc 2024-03-29T05:00:08.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:08 smithi181 ceph-mon[106466]: failed to encode map e455 with expected crc 2024-03-29T05:00:08.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:08 smithi181 ceph-mon[106466]: failed to encode map e455 with expected crc 2024-03-29T05:00:08.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:08 smithi181 ceph-mon[106466]: failed to encode map e455 with expected crc 2024-03-29T05:00:08.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:08 smithi181 ceph-mon[106466]: pgmap v344: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 6.5 KiB/s wr, 94 op/s 2024-03-29T05:00:08.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:08 smithi181 ceph-mon[106466]: osdmap e456: 8 total, 8 up, 8 in 2024-03-29T05:00:09.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:00:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:00:08] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.43.0" 2024-03-29T05:00:09.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:09 smithi187 ceph-mon[93536]: failed to encode map e456 with expected crc 2024-03-29T05:00:09.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:09 smithi187 ceph-mon[93536]: failed to encode map e456 with expected crc 2024-03-29T05:00:09.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:09 smithi187 ceph-mon[93536]: failed to encode map e456 with expected crc 2024-03-29T05:00:09.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:09 smithi187 ceph-mon[93536]: failed to encode map e456 with expected crc 2024-03-29T05:00:09.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:09 smithi187 ceph-mon[93536]: failed to encode map e456 with expected crc 2024-03-29T05:00:09.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:09 smithi187 ceph-mon[93536]: osdmap e457: 8 total, 8 up, 8 in 2024-03-29T05:00:09.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:09 smithi181 ceph-mon[101970]: failed to encode map e456 with expected crc 2024-03-29T05:00:09.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:09 smithi181 ceph-mon[101970]: failed to encode map e456 with expected crc 2024-03-29T05:00:09.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:09 smithi181 ceph-mon[101970]: failed to encode map e456 with expected crc 2024-03-29T05:00:09.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:09 smithi181 ceph-mon[101970]: failed to encode map e456 with expected crc 2024-03-29T05:00:09.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:09 smithi181 ceph-mon[101970]: failed to encode map e456 with expected crc 2024-03-29T05:00:09.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:09 smithi181 ceph-mon[101970]: osdmap e457: 8 total, 8 up, 8 in 2024-03-29T05:00:09.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:09 smithi181 ceph-mon[106466]: failed to encode map e456 with expected crc 2024-03-29T05:00:09.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:09 smithi181 ceph-mon[106466]: failed to encode map e456 with expected crc 2024-03-29T05:00:09.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:09 smithi181 ceph-mon[106466]: failed to encode map e456 with expected crc 2024-03-29T05:00:09.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:09 smithi181 ceph-mon[106466]: failed to encode map e456 with expected crc 2024-03-29T05:00:09.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:09 smithi181 ceph-mon[106466]: failed to encode map e456 with expected crc 2024-03-29T05:00:09.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:09 smithi181 ceph-mon[106466]: osdmap e457: 8 total, 8 up, 8 in 2024-03-29T05:00:10.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:10 smithi187 ceph-mon[93536]: failed to encode map e457 with expected crc 2024-03-29T05:00:10.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:10 smithi187 ceph-mon[93536]: failed to encode map e457 with expected crc 2024-03-29T05:00:10.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:10 smithi187 ceph-mon[93536]: failed to encode map e457 with expected crc 2024-03-29T05:00:10.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:10 smithi187 ceph-mon[93536]: failed to encode map e457 with expected crc 2024-03-29T05:00:10.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:10 smithi187 ceph-mon[93536]: failed to encode map e457 with expected crc 2024-03-29T05:00:10.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:10 smithi187 ceph-mon[93536]: pgmap v347: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 6.5 KiB/s wr, 94 op/s 2024-03-29T05:00:10.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:10 smithi187 ceph-mon[93536]: osdmap e458: 8 total, 8 up, 8 in 2024-03-29T05:00:10.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:10 smithi181 ceph-mon[106466]: failed to encode map e457 with expected crc 2024-03-29T05:00:10.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:10 smithi181 ceph-mon[106466]: failed to encode map e457 with expected crc 2024-03-29T05:00:10.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:10 smithi181 ceph-mon[106466]: failed to encode map e457 with expected crc 2024-03-29T05:00:10.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:10 smithi181 ceph-mon[106466]: failed to encode map e457 with expected crc 2024-03-29T05:00:10.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:10 smithi181 ceph-mon[106466]: failed to encode map e457 with expected crc 2024-03-29T05:00:10.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:10 smithi181 ceph-mon[106466]: pgmap v347: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 6.5 KiB/s wr, 94 op/s 2024-03-29T05:00:10.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:10 smithi181 ceph-mon[106466]: osdmap e458: 8 total, 8 up, 8 in 2024-03-29T05:00:10.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:10 smithi181 ceph-mon[101970]: failed to encode map e457 with expected crc 2024-03-29T05:00:10.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:10 smithi181 ceph-mon[101970]: failed to encode map e457 with expected crc 2024-03-29T05:00:10.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:10 smithi181 ceph-mon[101970]: failed to encode map e457 with expected crc 2024-03-29T05:00:10.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:10 smithi181 ceph-mon[101970]: failed to encode map e457 with expected crc 2024-03-29T05:00:10.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:10 smithi181 ceph-mon[101970]: failed to encode map e457 with expected crc 2024-03-29T05:00:10.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:10 smithi181 ceph-mon[101970]: pgmap v347: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 6.5 KiB/s wr, 94 op/s 2024-03-29T05:00:10.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:10 smithi181 ceph-mon[101970]: osdmap e458: 8 total, 8 up, 8 in 2024-03-29T05:00:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:11 smithi187 ceph-mon[93536]: failed to encode map e458 with expected crc 2024-03-29T05:00:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:11 smithi187 ceph-mon[93536]: failed to encode map e458 with expected crc 2024-03-29T05:00:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:11 smithi187 ceph-mon[93536]: failed to encode map e458 with expected crc 2024-03-29T05:00:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:11 smithi187 ceph-mon[93536]: failed to encode map e458 with expected crc 2024-03-29T05:00:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:11 smithi187 ceph-mon[93536]: failed to encode map e458 with expected crc 2024-03-29T05:00:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:00:11.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:11 smithi187 ceph-mon[93536]: osdmap e459: 8 total, 8 up, 8 in 2024-03-29T05:00:11.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:11 smithi181 ceph-mon[106466]: failed to encode map e458 with expected crc 2024-03-29T05:00:11.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:11 smithi181 ceph-mon[106466]: failed to encode map e458 with expected crc 2024-03-29T05:00:11.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:11 smithi181 ceph-mon[106466]: failed to encode map e458 with expected crc 2024-03-29T05:00:11.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:11 smithi181 ceph-mon[106466]: failed to encode map e458 with expected crc 2024-03-29T05:00:11.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:11 smithi181 ceph-mon[106466]: failed to encode map e458 with expected crc 2024-03-29T05:00:11.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:00:11.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:11 smithi181 ceph-mon[106466]: osdmap e459: 8 total, 8 up, 8 in 2024-03-29T05:00:11.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:11 smithi181 ceph-mon[101970]: failed to encode map e458 with expected crc 2024-03-29T05:00:11.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:11 smithi181 ceph-mon[101970]: failed to encode map e458 with expected crc 2024-03-29T05:00:11.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:11 smithi181 ceph-mon[101970]: failed to encode map e458 with expected crc 2024-03-29T05:00:11.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:11 smithi181 ceph-mon[101970]: failed to encode map e458 with expected crc 2024-03-29T05:00:11.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:11 smithi181 ceph-mon[101970]: failed to encode map e458 with expected crc 2024-03-29T05:00:11.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:00:11.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:11 smithi181 ceph-mon[101970]: osdmap e459: 8 total, 8 up, 8 in 2024-03-29T05:00:12.580 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.Snapshot (16060 ms) 2024-03-29T05:00:12.580 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.SnapshotRemoveOnDisable 2024-03-29T05:00:12.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:12 smithi187 ceph-mon[93536]: failed to encode map e459 with expected crc 2024-03-29T05:00:12.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:12 smithi187 ceph-mon[93536]: failed to encode map e459 with expected crc 2024-03-29T05:00:12.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:12 smithi187 ceph-mon[93536]: failed to encode map e459 with expected crc 2024-03-29T05:00:12.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:12 smithi187 ceph-mon[93536]: failed to encode map e459 with expected crc 2024-03-29T05:00:12.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:12 smithi187 ceph-mon[93536]: pgmap v350: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 255 B/s wr, 4 op/s 2024-03-29T05:00:12.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:12 smithi187 ceph-mon[93536]: osdmap e460: 8 total, 8 up, 8 in 2024-03-29T05:00:12.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:12 smithi181 ceph-mon[101970]: failed to encode map e459 with expected crc 2024-03-29T05:00:12.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:12 smithi181 ceph-mon[101970]: failed to encode map e459 with expected crc 2024-03-29T05:00:12.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:12 smithi181 ceph-mon[101970]: failed to encode map e459 with expected crc 2024-03-29T05:00:12.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:12 smithi181 ceph-mon[101970]: failed to encode map e459 with expected crc 2024-03-29T05:00:12.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:12 smithi181 ceph-mon[101970]: pgmap v350: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 255 B/s wr, 4 op/s 2024-03-29T05:00:12.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:12 smithi181 ceph-mon[101970]: osdmap e460: 8 total, 8 up, 8 in 2024-03-29T05:00:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:12 smithi181 ceph-mon[106466]: failed to encode map e459 with expected crc 2024-03-29T05:00:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:12 smithi181 ceph-mon[106466]: failed to encode map e459 with expected crc 2024-03-29T05:00:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:12 smithi181 ceph-mon[106466]: failed to encode map e459 with expected crc 2024-03-29T05:00:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:12 smithi181 ceph-mon[106466]: failed to encode map e459 with expected crc 2024-03-29T05:00:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:12 smithi181 ceph-mon[106466]: pgmap v350: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 255 B/s wr, 4 op/s 2024-03-29T05:00:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:12 smithi181 ceph-mon[106466]: osdmap e460: 8 total, 8 up, 8 in 2024-03-29T05:00:13.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:13 smithi187 ceph-mon[93536]: failed to encode map e460 with expected crc 2024-03-29T05:00:13.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:13 smithi187 ceph-mon[93536]: failed to encode map e460 with expected crc 2024-03-29T05:00:13.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:13 smithi187 ceph-mon[93536]: failed to encode map e460 with expected crc 2024-03-29T05:00:13.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:13 smithi187 ceph-mon[93536]: failed to encode map e460 with expected crc 2024-03-29T05:00:13.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:13 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/c899f714-348b-45e9-9b36-c91044cefb1b"}]: dispatch 2024-03-29T05:00:13.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:13 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/c899f714-348b-45e9-9b36-c91044cefb1b"}]': finished 2024-03-29T05:00:13.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:13 smithi181 ceph-mon[101970]: failed to encode map e460 with expected crc 2024-03-29T05:00:13.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:13 smithi181 ceph-mon[101970]: failed to encode map e460 with expected crc 2024-03-29T05:00:13.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:13 smithi181 ceph-mon[101970]: failed to encode map e460 with expected crc 2024-03-29T05:00:13.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:13 smithi181 ceph-mon[101970]: failed to encode map e460 with expected crc 2024-03-29T05:00:13.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:13 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/c899f714-348b-45e9-9b36-c91044cefb1b"}]: dispatch 2024-03-29T05:00:13.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:13 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/c899f714-348b-45e9-9b36-c91044cefb1b"}]': finished 2024-03-29T05:00:13.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:13 smithi181 ceph-mon[106466]: failed to encode map e460 with expected crc 2024-03-29T05:00:13.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:13 smithi181 ceph-mon[106466]: failed to encode map e460 with expected crc 2024-03-29T05:00:13.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:13 smithi181 ceph-mon[106466]: failed to encode map e460 with expected crc 2024-03-29T05:00:13.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:13 smithi181 ceph-mon[106466]: failed to encode map e460 with expected crc 2024-03-29T05:00:13.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:13 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/c899f714-348b-45e9-9b36-c91044cefb1b"}]: dispatch 2024-03-29T05:00:13.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:13 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/c899f714-348b-45e9-9b36-c91044cefb1b"}]': finished 2024-03-29T05:00:14.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:14 smithi187 ceph-mon[93536]: osdmap e461: 8 total, 8 up, 8 in 2024-03-29T05:00:14.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:14 smithi187 ceph-mon[93536]: failed to encode map e461 with expected crc 2024-03-29T05:00:14.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:14 smithi187 ceph-mon[93536]: failed to encode map e461 with expected crc 2024-03-29T05:00:14.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:14 smithi187 ceph-mon[93536]: pgmap v353: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 5.2 KiB/s wr, 76 op/s 2024-03-29T05:00:14.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:14 smithi181 ceph-mon[101970]: osdmap e461: 8 total, 8 up, 8 in 2024-03-29T05:00:14.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:14 smithi181 ceph-mon[101970]: failed to encode map e461 with expected crc 2024-03-29T05:00:14.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:14 smithi181 ceph-mon[101970]: failed to encode map e461 with expected crc 2024-03-29T05:00:14.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:14 smithi181 ceph-mon[101970]: pgmap v353: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 5.2 KiB/s wr, 76 op/s 2024-03-29T05:00:14.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:14 smithi181 ceph-mon[106466]: osdmap e461: 8 total, 8 up, 8 in 2024-03-29T05:00:14.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:14 smithi181 ceph-mon[106466]: failed to encode map e461 with expected crc 2024-03-29T05:00:14.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:14 smithi181 ceph-mon[106466]: failed to encode map e461 with expected crc 2024-03-29T05:00:14.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:14 smithi181 ceph-mon[106466]: pgmap v353: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 5.2 KiB/s wr, 76 op/s 2024-03-29T05:00:15.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:15 smithi187 ceph-mon[93536]: failed to encode map e462 with expected crc 2024-03-29T05:00:15.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:15 smithi187 ceph-mon[93536]: failed to encode map e462 with expected crc 2024-03-29T05:00:15.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:15 smithi187 ceph-mon[93536]: osdmap e462: 8 total, 8 up, 8 in 2024-03-29T05:00:15.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:15 smithi187 ceph-mon[93536]: failed to encode map e462 with expected crc 2024-03-29T05:00:15.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:15 smithi187 ceph-mon[93536]: failed to encode map e462 with expected crc 2024-03-29T05:00:15.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:15 smithi181 ceph-mon[106466]: failed to encode map e462 with expected crc 2024-03-29T05:00:15.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:15 smithi181 ceph-mon[106466]: failed to encode map e462 with expected crc 2024-03-29T05:00:15.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:15 smithi181 ceph-mon[106466]: osdmap e462: 8 total, 8 up, 8 in 2024-03-29T05:00:15.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:15 smithi181 ceph-mon[106466]: failed to encode map e462 with expected crc 2024-03-29T05:00:15.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:15 smithi181 ceph-mon[106466]: failed to encode map e462 with expected crc 2024-03-29T05:00:15.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:15 smithi181 ceph-mon[101970]: failed to encode map e462 with expected crc 2024-03-29T05:00:15.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:15 smithi181 ceph-mon[101970]: failed to encode map e462 with expected crc 2024-03-29T05:00:15.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:15 smithi181 ceph-mon[101970]: osdmap e462: 8 total, 8 up, 8 in 2024-03-29T05:00:15.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:15 smithi181 ceph-mon[101970]: failed to encode map e462 with expected crc 2024-03-29T05:00:15.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:15 smithi181 ceph-mon[101970]: failed to encode map e462 with expected crc 2024-03-29T05:00:16.687 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.SnapshotRemoveOnDisable (4109 ms) 2024-03-29T05:00:16.687 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.SnapshotUnlinkPeer 2024-03-29T05:00:16.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:16 smithi187 ceph-mon[93536]: failed to encode map e463 with expected crc 2024-03-29T05:00:16.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:16 smithi187 ceph-mon[93536]: failed to encode map e463 with expected crc 2024-03-29T05:00:16.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:16 smithi187 ceph-mon[93536]: osdmap e463: 8 total, 8 up, 8 in 2024-03-29T05:00:16.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:16 smithi187 ceph-mon[93536]: failed to encode map e463 with expected crc 2024-03-29T05:00:16.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:16 smithi187 ceph-mon[93536]: pgmap v356: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 5.0 KiB/s wr, 71 op/s 2024-03-29T05:00:16.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:16 smithi181 ceph-mon[106466]: failed to encode map e463 with expected crc 2024-03-29T05:00:16.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:16 smithi181 ceph-mon[106466]: failed to encode map e463 with expected crc 2024-03-29T05:00:16.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:16 smithi181 ceph-mon[106466]: osdmap e463: 8 total, 8 up, 8 in 2024-03-29T05:00:16.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:16 smithi181 ceph-mon[106466]: failed to encode map e463 with expected crc 2024-03-29T05:00:16.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:16 smithi181 ceph-mon[106466]: pgmap v356: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 5.0 KiB/s wr, 71 op/s 2024-03-29T05:00:16.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:16 smithi181 ceph-mon[101970]: failed to encode map e463 with expected crc 2024-03-29T05:00:16.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:16 smithi181 ceph-mon[101970]: failed to encode map e463 with expected crc 2024-03-29T05:00:16.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:16 smithi181 ceph-mon[101970]: osdmap e463: 8 total, 8 up, 8 in 2024-03-29T05:00:16.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:16 smithi181 ceph-mon[101970]: failed to encode map e463 with expected crc 2024-03-29T05:00:16.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:16 smithi181 ceph-mon[101970]: pgmap v356: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 5.0 KiB/s wr, 71 op/s 2024-03-29T05:00:17.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:17 smithi187 ceph-mon[93536]: failed to encode map e464 with expected crc 2024-03-29T05:00:17.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:17 smithi187 ceph-mon[93536]: failed to encode map e464 with expected crc 2024-03-29T05:00:17.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:17 smithi187 ceph-mon[93536]: osdmap e464: 8 total, 8 up, 8 in 2024-03-29T05:00:17.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:17 smithi187 ceph-mon[93536]: failed to encode map e464 with expected crc 2024-03-29T05:00:17.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:17 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b54e69fe-c9e7-4e5e-97f0-76d57d92d8d7"}]: dispatch 2024-03-29T05:00:17.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:17 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b54e69fe-c9e7-4e5e-97f0-76d57d92d8d7"}]': finished 2024-03-29T05:00:17.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:17 smithi187 ceph-mon[93536]: failed to encode map e465 with expected crc 2024-03-29T05:00:17.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:17 smithi187 ceph-mon[93536]: osdmap e465: 8 total, 8 up, 8 in 2024-03-29T05:00:17.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:17 smithi187 ceph-mon[93536]: failed to encode map e465 with expected crc 2024-03-29T05:00:17.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[106466]: failed to encode map e464 with expected crc 2024-03-29T05:00:17.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[106466]: failed to encode map e464 with expected crc 2024-03-29T05:00:17.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[106466]: osdmap e464: 8 total, 8 up, 8 in 2024-03-29T05:00:17.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[106466]: failed to encode map e464 with expected crc 2024-03-29T05:00:17.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b54e69fe-c9e7-4e5e-97f0-76d57d92d8d7"}]: dispatch 2024-03-29T05:00:17.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b54e69fe-c9e7-4e5e-97f0-76d57d92d8d7"}]': finished 2024-03-29T05:00:17.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[106466]: failed to encode map e465 with expected crc 2024-03-29T05:00:17.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[106466]: osdmap e465: 8 total, 8 up, 8 in 2024-03-29T05:00:17.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[106466]: failed to encode map e465 with expected crc 2024-03-29T05:00:17.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[101970]: failed to encode map e464 with expected crc 2024-03-29T05:00:17.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[101970]: failed to encode map e464 with expected crc 2024-03-29T05:00:17.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[101970]: osdmap e464: 8 total, 8 up, 8 in 2024-03-29T05:00:17.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[101970]: failed to encode map e464 with expected crc 2024-03-29T05:00:17.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b54e69fe-c9e7-4e5e-97f0-76d57d92d8d7"}]: dispatch 2024-03-29T05:00:17.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b54e69fe-c9e7-4e5e-97f0-76d57d92d8d7"}]': finished 2024-03-29T05:00:17.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[101970]: failed to encode map e465 with expected crc 2024-03-29T05:00:17.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[101970]: osdmap e465: 8 total, 8 up, 8 in 2024-03-29T05:00:17.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:17 smithi181 ceph-mon[101970]: failed to encode map e465 with expected crc 2024-03-29T05:00:18.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:18 smithi187 ceph-mon[93536]: failed to encode map e465 with expected crc 2024-03-29T05:00:18.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:18 smithi187 ceph-mon[93536]: failed to encode map e465 with expected crc 2024-03-29T05:00:18.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:18 smithi187 ceph-mon[93536]: failed to encode map e465 with expected crc 2024-03-29T05:00:18.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:18 smithi187 ceph-mon[93536]: pgmap v359: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 113 KiB/s rd, 12 KiB/s wr, 160 op/s 2024-03-29T05:00:18.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:18 smithi187 ceph-mon[93536]: failed to encode map e466 with expected crc 2024-03-29T05:00:18.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:18 smithi187 ceph-mon[93536]: osdmap e466: 8 total, 8 up, 8 in 2024-03-29T05:00:18.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:18 smithi187 ceph-mon[93536]: failed to encode map e466 with expected crc 2024-03-29T05:00:18.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:18 smithi181 ceph-mon[101970]: failed to encode map e465 with expected crc 2024-03-29T05:00:18.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:18 smithi181 ceph-mon[101970]: failed to encode map e465 with expected crc 2024-03-29T05:00:18.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:18 smithi181 ceph-mon[101970]: failed to encode map e465 with expected crc 2024-03-29T05:00:18.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:18 smithi181 ceph-mon[101970]: pgmap v359: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 113 KiB/s rd, 12 KiB/s wr, 160 op/s 2024-03-29T05:00:18.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:18 smithi181 ceph-mon[101970]: failed to encode map e466 with expected crc 2024-03-29T05:00:18.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:18 smithi181 ceph-mon[101970]: osdmap e466: 8 total, 8 up, 8 in 2024-03-29T05:00:18.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:18 smithi181 ceph-mon[101970]: failed to encode map e466 with expected crc 2024-03-29T05:00:18.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:18 smithi181 ceph-mon[106466]: failed to encode map e465 with expected crc 2024-03-29T05:00:18.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:18 smithi181 ceph-mon[106466]: failed to encode map e465 with expected crc 2024-03-29T05:00:18.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:18 smithi181 ceph-mon[106466]: failed to encode map e465 with expected crc 2024-03-29T05:00:18.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:18 smithi181 ceph-mon[106466]: pgmap v359: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 113 KiB/s rd, 12 KiB/s wr, 160 op/s 2024-03-29T05:00:18.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:18 smithi181 ceph-mon[106466]: failed to encode map e466 with expected crc 2024-03-29T05:00:18.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:18 smithi181 ceph-mon[106466]: osdmap e466: 8 total, 8 up, 8 in 2024-03-29T05:00:18.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:18 smithi181 ceph-mon[106466]: failed to encode map e466 with expected crc 2024-03-29T05:00:19.454 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:00:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:00:18] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.43.0" 2024-03-29T05:00:19.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:19 smithi187 ceph-mon[93536]: failed to encode map e466 with expected crc 2024-03-29T05:00:19.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:19 smithi187 ceph-mon[93536]: failed to encode map e466 with expected crc 2024-03-29T05:00:19.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:19 smithi187 ceph-mon[93536]: failed to encode map e466 with expected crc 2024-03-29T05:00:19.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:19 smithi187 ceph-mon[93536]: osdmap e467: 8 total, 8 up, 8 in 2024-03-29T05:00:19.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:19 smithi187 ceph-mon[93536]: failed to encode map e467 with expected crc 2024-03-29T05:00:19.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:19 smithi187 ceph-mon[93536]: failed to encode map e467 with expected crc 2024-03-29T05:00:19.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:19 smithi181 ceph-mon[106466]: failed to encode map e466 with expected crc 2024-03-29T05:00:19.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:19 smithi181 ceph-mon[106466]: failed to encode map e466 with expected crc 2024-03-29T05:00:19.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:19 smithi181 ceph-mon[106466]: failed to encode map e466 with expected crc 2024-03-29T05:00:19.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:19 smithi181 ceph-mon[106466]: osdmap e467: 8 total, 8 up, 8 in 2024-03-29T05:00:19.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:19 smithi181 ceph-mon[106466]: failed to encode map e467 with expected crc 2024-03-29T05:00:19.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:19 smithi181 ceph-mon[106466]: failed to encode map e467 with expected crc 2024-03-29T05:00:19.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:19 smithi181 ceph-mon[101970]: failed to encode map e466 with expected crc 2024-03-29T05:00:19.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:19 smithi181 ceph-mon[101970]: failed to encode map e466 with expected crc 2024-03-29T05:00:19.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:19 smithi181 ceph-mon[101970]: failed to encode map e466 with expected crc 2024-03-29T05:00:19.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:19 smithi181 ceph-mon[101970]: osdmap e467: 8 total, 8 up, 8 in 2024-03-29T05:00:19.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:19 smithi181 ceph-mon[101970]: failed to encode map e467 with expected crc 2024-03-29T05:00:19.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:19 smithi181 ceph-mon[101970]: failed to encode map e467 with expected crc 2024-03-29T05:00:20.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:20 smithi187 ceph-mon[93536]: failed to encode map e467 with expected crc 2024-03-29T05:00:20.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:20 smithi187 ceph-mon[93536]: failed to encode map e467 with expected crc 2024-03-29T05:00:20.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:20 smithi187 ceph-mon[93536]: failed to encode map e467 with expected crc 2024-03-29T05:00:20.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:20 smithi187 ceph-mon[93536]: pgmap v362: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 113 KiB/s rd, 12 KiB/s wr, 160 op/s 2024-03-29T05:00:20.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:20 smithi187 ceph-mon[93536]: osdmap e468: 8 total, 8 up, 8 in 2024-03-29T05:00:20.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:20 smithi187 ceph-mon[93536]: failed to encode map e468 with expected crc 2024-03-29T05:00:20.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:20 smithi187 ceph-mon[93536]: failed to encode map e468 with expected crc 2024-03-29T05:00:20.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:20 smithi181 ceph-mon[106466]: failed to encode map e467 with expected crc 2024-03-29T05:00:20.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:20 smithi181 ceph-mon[106466]: failed to encode map e467 with expected crc 2024-03-29T05:00:20.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:20 smithi181 ceph-mon[106466]: failed to encode map e467 with expected crc 2024-03-29T05:00:20.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:20 smithi181 ceph-mon[106466]: pgmap v362: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 113 KiB/s rd, 12 KiB/s wr, 160 op/s 2024-03-29T05:00:20.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:20 smithi181 ceph-mon[106466]: osdmap e468: 8 total, 8 up, 8 in 2024-03-29T05:00:20.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:20 smithi181 ceph-mon[106466]: failed to encode map e468 with expected crc 2024-03-29T05:00:20.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:20 smithi181 ceph-mon[106466]: failed to encode map e468 with expected crc 2024-03-29T05:00:20.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:20 smithi181 ceph-mon[101970]: failed to encode map e467 with expected crc 2024-03-29T05:00:20.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:20 smithi181 ceph-mon[101970]: failed to encode map e467 with expected crc 2024-03-29T05:00:20.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:20 smithi181 ceph-mon[101970]: failed to encode map e467 with expected crc 2024-03-29T05:00:20.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:20 smithi181 ceph-mon[101970]: pgmap v362: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 113 KiB/s rd, 12 KiB/s wr, 160 op/s 2024-03-29T05:00:20.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:20 smithi181 ceph-mon[101970]: osdmap e468: 8 total, 8 up, 8 in 2024-03-29T05:00:20.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:20 smithi181 ceph-mon[101970]: failed to encode map e468 with expected crc 2024-03-29T05:00:20.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:20 smithi181 ceph-mon[101970]: failed to encode map e468 with expected crc 2024-03-29T05:00:21.508 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade resume' 2024-03-29T05:00:21.782 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:00:21.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:21 smithi187 ceph-mon[93536]: failed to encode map e468 with expected crc 2024-03-29T05:00:21.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:21 smithi187 ceph-mon[93536]: failed to encode map e468 with expected crc 2024-03-29T05:00:21.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:21 smithi187 ceph-mon[93536]: osdmap e469: 8 total, 8 up, 8 in 2024-03-29T05:00:21.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:21 smithi187 ceph-mon[93536]: failed to encode map e469 with expected crc 2024-03-29T05:00:21.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:21 smithi187 ceph-mon[93536]: failed to encode map e469 with expected crc 2024-03-29T05:00:21.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:21 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/aa7c4ac3-bb12-47df-a89d-0e08eb320240"}]: dispatch 2024-03-29T05:00:21.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:21 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/aa7c4ac3-bb12-47df-a89d-0e08eb320240"}]': finished 2024-03-29T05:00:21.904 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:21 smithi181 ceph-mon[101970]: failed to encode map e468 with expected crc 2024-03-29T05:00:21.904 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:21 smithi181 ceph-mon[101970]: failed to encode map e468 with expected crc 2024-03-29T05:00:21.904 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:21 smithi181 ceph-mon[101970]: osdmap e469: 8 total, 8 up, 8 in 2024-03-29T05:00:21.904 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:21 smithi181 ceph-mon[101970]: failed to encode map e469 with expected crc 2024-03-29T05:00:21.904 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:21 smithi181 ceph-mon[101970]: failed to encode map e469 with expected crc 2024-03-29T05:00:21.904 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:21 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/aa7c4ac3-bb12-47df-a89d-0e08eb320240"}]: dispatch 2024-03-29T05:00:21.904 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:21 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/aa7c4ac3-bb12-47df-a89d-0e08eb320240"}]': finished 2024-03-29T05:00:21.904 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:21 smithi181 ceph-mon[106466]: failed to encode map e468 with expected crc 2024-03-29T05:00:21.904 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:21 smithi181 ceph-mon[106466]: failed to encode map e468 with expected crc 2024-03-29T05:00:21.904 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:21 smithi181 ceph-mon[106466]: osdmap e469: 8 total, 8 up, 8 in 2024-03-29T05:00:21.905 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:21 smithi181 ceph-mon[106466]: failed to encode map e469 with expected crc 2024-03-29T05:00:21.905 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:21 smithi181 ceph-mon[106466]: failed to encode map e469 with expected crc 2024-03-29T05:00:21.905 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:21 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/aa7c4ac3-bb12-47df-a89d-0e08eb320240"}]: dispatch 2024-03-29T05:00:21.905 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:21 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/aa7c4ac3-bb12-47df-a89d-0e08eb320240"}]': finished 2024-03-29T05:00:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:22 smithi187 ceph-mon[93536]: failed to encode map e469 with expected crc 2024-03-29T05:00:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:22 smithi187 ceph-mon[93536]: failed to encode map e469 with expected crc 2024-03-29T05:00:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:22 smithi187 ceph-mon[93536]: pgmap v365: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 1023 B/s wr, 2 op/s 2024-03-29T05:00:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:22 smithi187 ceph-mon[93536]: osdmap e470: 8 total, 8 up, 8 in 2024-03-29T05:00:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:22 smithi187 ceph-mon[93536]: failed to encode map e470 with expected crc 2024-03-29T05:00:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:22 smithi187 ceph-mon[93536]: failed to encode map e470 with expected crc 2024-03-29T05:00:22.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:22 smithi181 ceph-mon[106466]: failed to encode map e469 with expected crc 2024-03-29T05:00:22.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:22 smithi181 ceph-mon[106466]: failed to encode map e469 with expected crc 2024-03-29T05:00:22.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:22 smithi181 ceph-mon[106466]: pgmap v365: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 1023 B/s wr, 2 op/s 2024-03-29T05:00:22.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:22 smithi181 ceph-mon[106466]: osdmap e470: 8 total, 8 up, 8 in 2024-03-29T05:00:22.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:22 smithi181 ceph-mon[106466]: failed to encode map e470 with expected crc 2024-03-29T05:00:22.949 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:22 smithi181 ceph-mon[106466]: failed to encode map e470 with expected crc 2024-03-29T05:00:22.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:22 smithi181 ceph-mon[101970]: failed to encode map e469 with expected crc 2024-03-29T05:00:22.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:22 smithi181 ceph-mon[101970]: failed to encode map e469 with expected crc 2024-03-29T05:00:22.949 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:22 smithi181 ceph-mon[101970]: pgmap v365: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 1023 B/s wr, 2 op/s 2024-03-29T05:00:22.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:22 smithi181 ceph-mon[101970]: osdmap e470: 8 total, 8 up, 8 in 2024-03-29T05:00:22.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:22 smithi181 ceph-mon[101970]: failed to encode map e470 with expected crc 2024-03-29T05:00:22.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:22 smithi181 ceph-mon[101970]: failed to encode map e470 with expected crc 2024-03-29T05:00:23.683 INFO:teuthology.orchestra.run.smithi181.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:00:23.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:23 smithi187 ceph-mon[93536]: failed to encode map e470 with expected crc 2024-03-29T05:00:23.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:23 smithi187 ceph-mon[93536]: failed to encode map e470 with expected crc 2024-03-29T05:00:23.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:23 smithi187 ceph-mon[93536]: failed to encode map e470 with expected crc 2024-03-29T05:00:23.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:23 smithi187 ceph-mon[93536]: osdmap e471: 8 total, 8 up, 8 in 2024-03-29T05:00:23.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:23 smithi187 ceph-mon[93536]: failed to encode map e471 with expected crc 2024-03-29T05:00:23.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:23 smithi181 ceph-mon[106466]: failed to encode map e470 with expected crc 2024-03-29T05:00:23.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:23 smithi181 ceph-mon[106466]: failed to encode map e470 with expected crc 2024-03-29T05:00:23.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:23 smithi181 ceph-mon[106466]: failed to encode map e470 with expected crc 2024-03-29T05:00:23.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:23 smithi181 ceph-mon[106466]: osdmap e471: 8 total, 8 up, 8 in 2024-03-29T05:00:23.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:23 smithi181 ceph-mon[106466]: failed to encode map e471 with expected crc 2024-03-29T05:00:23.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:23 smithi181 ceph-mon[101970]: failed to encode map e470 with expected crc 2024-03-29T05:00:23.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:23 smithi181 ceph-mon[101970]: failed to encode map e470 with expected crc 2024-03-29T05:00:23.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:23 smithi181 ceph-mon[101970]: failed to encode map e470 with expected crc 2024-03-29T05:00:23.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:23 smithi181 ceph-mon[101970]: osdmap e471: 8 total, 8 up, 8 in 2024-03-29T05:00:23.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:23 smithi181 ceph-mon[101970]: failed to encode map e471 with expected crc 2024-03-29T05:00:24.694 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'echo wait for half of osds to upgrade' 2024-03-29T05:00:24.934 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: failed to encode map e471 with expected crc 2024-03-29T05:00:24.934 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: failed to encode map e471 with expected crc 2024-03-29T05:00:24.934 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: failed to encode map e471 with expected crc 2024-03-29T05:00:24.934 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T05:00:24.934 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:00:24.935 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:24.935 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:00:24.935 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:00:24.935 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:00:24.935 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: pgmap v368: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 169 KiB/s rd, 12 KiB/s wr, 220 op/s 2024-03-29T05:00:24.935 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: pgmap v369: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 200 KiB/s rd, 14 KiB/s wr, 260 op/s 2024-03-29T05:00:24.935 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:24.935 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:00:24.935 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:24.935 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:24.935 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:24.935 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T05:00:24.935 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T05:00:24.935 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: Upgrade: osd.1 is safe to restart 2024-03-29T05:00:24.936 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: failed to encode map e472 with expected crc 2024-03-29T05:00:24.936 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[106466]: osdmap e472: 8 total, 8 up, 8 in 2024-03-29T05:00:24.936 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: failed to encode map e471 with expected crc 2024-03-29T05:00:24.936 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: failed to encode map e471 with expected crc 2024-03-29T05:00:24.936 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: failed to encode map e471 with expected crc 2024-03-29T05:00:24.936 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T05:00:24.936 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:00:24.936 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:24.936 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:00:24.937 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:00:24.937 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:00:24.937 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: pgmap v368: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 169 KiB/s rd, 12 KiB/s wr, 220 op/s 2024-03-29T05:00:24.937 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: pgmap v369: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 200 KiB/s rd, 14 KiB/s wr, 260 op/s 2024-03-29T05:00:24.937 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:24.937 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:00:24.937 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:24.937 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:24.937 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:24.937 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T05:00:24.937 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T05:00:24.937 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: Upgrade: osd.1 is safe to restart 2024-03-29T05:00:24.937 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: failed to encode map e472 with expected crc 2024-03-29T05:00:24.937 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:24 smithi181 ceph-mon[101970]: osdmap e472: 8 total, 8 up, 8 in 2024-03-29T05:00:25.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: failed to encode map e471 with expected crc 2024-03-29T05:00:25.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: failed to encode map e471 with expected crc 2024-03-29T05:00:25.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: failed to encode map e471 with expected crc 2024-03-29T05:00:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T05:00:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:00:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:00:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:00:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:00:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: pgmap v368: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 169 KiB/s rd, 12 KiB/s wr, 220 op/s 2024-03-29T05:00:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: pgmap v369: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 200 KiB/s rd, 14 KiB/s wr, 260 op/s 2024-03-29T05:00:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:00:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T05:00:25.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T05:00:25.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: Upgrade: osd.1 is safe to restart 2024-03-29T05:00:25.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: failed to encode map e472 with expected crc 2024-03-29T05:00:25.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:24 smithi187 ceph-mon[93536]: osdmap e472: 8 total, 8 up, 8 in 2024-03-29T05:00:25.451 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:00:25.841 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:25 smithi181 ceph-mon[101970]: failed to encode map e472 with expected crc 2024-03-29T05:00:25.841 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:25 smithi181 ceph-mon[101970]: failed to encode map e472 with expected crc 2024-03-29T05:00:25.841 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:25 smithi181 ceph-mon[101970]: failed to encode map e472 with expected crc 2024-03-29T05:00:25.841 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:25 smithi181 ceph-mon[101970]: failed to encode map e472 with expected crc 2024-03-29T05:00:25.841 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:25 smithi181 ceph-mon[101970]: failed to encode map e473 with expected crc 2024-03-29T05:00:25.841 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:25 smithi181 ceph-mon[101970]: osdmap e473: 8 total, 8 up, 8 in 2024-03-29T05:00:25.842 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:25 smithi181 ceph-mon[101970]: failed to encode map e473 with expected crc 2024-03-29T05:00:25.842 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:25 smithi181 ceph-mon[106466]: failed to encode map e472 with expected crc 2024-03-29T05:00:25.842 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:25 smithi181 ceph-mon[106466]: failed to encode map e472 with expected crc 2024-03-29T05:00:25.842 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:25 smithi181 ceph-mon[106466]: failed to encode map e472 with expected crc 2024-03-29T05:00:25.842 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:25 smithi181 ceph-mon[106466]: failed to encode map e472 with expected crc 2024-03-29T05:00:25.842 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:25 smithi181 ceph-mon[106466]: failed to encode map e473 with expected crc 2024-03-29T05:00:25.842 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:25 smithi181 ceph-mon[106466]: osdmap e473: 8 total, 8 up, 8 in 2024-03-29T05:00:25.842 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:25 smithi181 ceph-mon[106466]: failed to encode map e473 with expected crc 2024-03-29T05:00:26.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:25 smithi187 ceph-mon[93536]: failed to encode map e472 with expected crc 2024-03-29T05:00:26.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:25 smithi187 ceph-mon[93536]: failed to encode map e472 with expected crc 2024-03-29T05:00:26.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:25 smithi187 ceph-mon[93536]: failed to encode map e472 with expected crc 2024-03-29T05:00:26.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:25 smithi187 ceph-mon[93536]: failed to encode map e472 with expected crc 2024-03-29T05:00:26.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:25 smithi187 ceph-mon[93536]: failed to encode map e473 with expected crc 2024-03-29T05:00:26.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:25 smithi187 ceph-mon[93536]: osdmap e473: 8 total, 8 up, 8 in 2024-03-29T05:00:26.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:25 smithi187 ceph-mon[93536]: failed to encode map e473 with expected crc 2024-03-29T05:00:26.614 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.SnapshotUnlinkPeer (9927 ms) 2024-03-29T05:00:26.614 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.SnapshotImageState 2024-03-29T05:00:27.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: failed to encode map e473 with expected crc 2024-03-29T05:00:27.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: failed to encode map e473 with expected crc 2024-03-29T05:00:27.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: pgmap v372: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:00:27.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: failed to encode map e473 with expected crc 2024-03-29T05:00:27.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: Upgrade: Updating osd.1 2024-03-29T05:00:27.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:27.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T05:00:27.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:00:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: Deploying daemon osd.1 on smithi181 2024-03-29T05:00:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:00:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: osdmap e474: 8 total, 8 up, 8 in 2024-03-29T05:00:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: failed to encode map e474 with expected crc 2024-03-29T05:00:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: failed to encode map e474 with expected crc 2024-03-29T05:00:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/a269501e-96fb-4226-aff3-22a003db7a80"}]: dispatch 2024-03-29T05:00:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/a269501e-96fb-4226-aff3-22a003db7a80"}]': finished 2024-03-29T05:00:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/67763465-1907-4751-8003-d23983beaa93"}]: dispatch 2024-03-29T05:00:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/67763465-1907-4751-8003-d23983beaa93"}]': finished 2024-03-29T05:00:27.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: failed to encode map e473 with expected crc 2024-03-29T05:00:27.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: failed to encode map e473 with expected crc 2024-03-29T05:00:27.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: pgmap v372: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:00:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: failed to encode map e473 with expected crc 2024-03-29T05:00:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: Upgrade: Updating osd.1 2024-03-29T05:00:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T05:00:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:00:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: Deploying daemon osd.1 on smithi181 2024-03-29T05:00:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:00:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: osdmap e474: 8 total, 8 up, 8 in 2024-03-29T05:00:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: failed to encode map e474 with expected crc 2024-03-29T05:00:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: failed to encode map e474 with expected crc 2024-03-29T05:00:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/a269501e-96fb-4226-aff3-22a003db7a80"}]: dispatch 2024-03-29T05:00:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/a269501e-96fb-4226-aff3-22a003db7a80"}]': finished 2024-03-29T05:00:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/67763465-1907-4751-8003-d23983beaa93"}]: dispatch 2024-03-29T05:00:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:26 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/67763465-1907-4751-8003-d23983beaa93"}]': finished 2024-03-29T05:00:27.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: failed to encode map e473 with expected crc 2024-03-29T05:00:27.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: failed to encode map e473 with expected crc 2024-03-29T05:00:27.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: pgmap v372: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:00:27.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: failed to encode map e473 with expected crc 2024-03-29T05:00:27.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: Upgrade: Updating osd.1 2024-03-29T05:00:27.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:27.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T05:00:27.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:00:27.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: Deploying daemon osd.1 on smithi181 2024-03-29T05:00:27.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:00:27.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: osdmap e474: 8 total, 8 up, 8 in 2024-03-29T05:00:27.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: failed to encode map e474 with expected crc 2024-03-29T05:00:27.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: failed to encode map e474 with expected crc 2024-03-29T05:00:27.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/a269501e-96fb-4226-aff3-22a003db7a80"}]: dispatch 2024-03-29T05:00:27.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/a269501e-96fb-4226-aff3-22a003db7a80"}]': finished 2024-03-29T05:00:27.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/67763465-1907-4751-8003-d23983beaa93"}]: dispatch 2024-03-29T05:00:27.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:26 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/67763465-1907-4751-8003-d23983beaa93"}]': finished 2024-03-29T05:00:28.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:27 smithi181 ceph-mon[101970]: failed to encode map e474 with expected crc 2024-03-29T05:00:28.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:27 smithi181 ceph-mon[101970]: failed to encode map e474 with expected crc 2024-03-29T05:00:28.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:27 smithi181 ceph-mon[101970]: failed to encode map e474 with expected crc 2024-03-29T05:00:28.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:27 smithi181 ceph-mon[101970]: osdmap e475: 8 total, 8 up, 8 in 2024-03-29T05:00:28.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:27 smithi181 ceph-mon[101970]: failed to encode map e475 with expected crc 2024-03-29T05:00:28.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:27 smithi181 ceph-mon[106466]: failed to encode map e474 with expected crc 2024-03-29T05:00:28.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:27 smithi181 ceph-mon[106466]: failed to encode map e474 with expected crc 2024-03-29T05:00:28.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:27 smithi181 ceph-mon[106466]: failed to encode map e474 with expected crc 2024-03-29T05:00:28.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:27 smithi181 ceph-mon[106466]: osdmap e475: 8 total, 8 up, 8 in 2024-03-29T05:00:28.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:27 smithi181 ceph-mon[106466]: failed to encode map e475 with expected crc 2024-03-29T05:00:28.285 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:27 smithi187 ceph-mon[93536]: failed to encode map e474 with expected crc 2024-03-29T05:00:28.286 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:27 smithi187 ceph-mon[93536]: failed to encode map e474 with expected crc 2024-03-29T05:00:28.286 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:27 smithi187 ceph-mon[93536]: failed to encode map e474 with expected crc 2024-03-29T05:00:28.286 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:27 smithi187 ceph-mon[93536]: osdmap e475: 8 total, 8 up, 8 in 2024-03-29T05:00:28.286 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:27 smithi187 ceph-mon[93536]: failed to encode map e475 with expected crc 2024-03-29T05:00:28.810 INFO:teuthology.orchestra.run.smithi181.stdout:wait for half of osds to upgrade 2024-03-29T05:00:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:28 smithi181 ceph-mon[106466]: failed to encode map e475 with expected crc 2024-03-29T05:00:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:28 smithi181 ceph-mon[106466]: failed to encode map e475 with expected crc 2024-03-29T05:00:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:28 smithi181 ceph-mon[106466]: failed to encode map e475 with expected crc 2024-03-29T05:00:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:28 smithi181 ceph-mon[106466]: pgmap v375: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 7.7 KiB/s wr, 136 op/s 2024-03-29T05:00:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:28 smithi181 ceph-mon[106466]: osdmap e476: 8 total, 8 up, 8 in 2024-03-29T05:00:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:28 smithi181 ceph-mon[106466]: failed to encode map e476 with expected crc 2024-03-29T05:00:28.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:28 smithi181 ceph-mon[101970]: failed to encode map e475 with expected crc 2024-03-29T05:00:28.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:28 smithi181 ceph-mon[101970]: failed to encode map e475 with expected crc 2024-03-29T05:00:28.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:28 smithi181 ceph-mon[101970]: failed to encode map e475 with expected crc 2024-03-29T05:00:28.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:28 smithi181 ceph-mon[101970]: pgmap v375: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 7.7 KiB/s wr, 136 op/s 2024-03-29T05:00:28.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:28 smithi181 ceph-mon[101970]: osdmap e476: 8 total, 8 up, 8 in 2024-03-29T05:00:28.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:28 smithi181 ceph-mon[101970]: failed to encode map e476 with expected crc 2024-03-29T05:00:29.255 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:00:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:00:28] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.43.0" 2024-03-29T05:00:29.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:28 smithi187 ceph-mon[93536]: failed to encode map e475 with expected crc 2024-03-29T05:00:29.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:28 smithi187 ceph-mon[93536]: failed to encode map e475 with expected crc 2024-03-29T05:00:29.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:28 smithi187 ceph-mon[93536]: failed to encode map e475 with expected crc 2024-03-29T05:00:29.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:28 smithi187 ceph-mon[93536]: pgmap v375: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 7.7 KiB/s wr, 136 op/s 2024-03-29T05:00:29.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:28 smithi187 ceph-mon[93536]: osdmap e476: 8 total, 8 up, 8 in 2024-03-29T05:00:29.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:28 smithi187 ceph-mon[93536]: failed to encode map e476 with expected crc 2024-03-29T05:00:29.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-03-29T05:00:30.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:29 smithi181 ceph-mon[106466]: failed to encode map e476 with expected crc 2024-03-29T05:00:30.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:29 smithi181 ceph-mon[106466]: failed to encode map e476 with expected crc 2024-03-29T05:00:30.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:29 smithi181 ceph-mon[106466]: failed to encode map e476 with expected crc 2024-03-29T05:00:30.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:29 smithi181 ceph-mon[106466]: osdmap e477: 8 total, 8 up, 8 in 2024-03-29T05:00:30.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:29 smithi181 ceph-mon[106466]: failed to encode map e477 with expected crc 2024-03-29T05:00:30.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:29 smithi181 ceph-mon[101970]: failed to encode map e476 with expected crc 2024-03-29T05:00:30.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:29 smithi181 ceph-mon[101970]: failed to encode map e476 with expected crc 2024-03-29T05:00:30.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:29 smithi181 ceph-mon[101970]: failed to encode map e476 with expected crc 2024-03-29T05:00:30.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:29 smithi181 ceph-mon[101970]: osdmap e477: 8 total, 8 up, 8 in 2024-03-29T05:00:30.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:29 smithi181 ceph-mon[101970]: failed to encode map e477 with expected crc 2024-03-29T05:00:30.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:29 smithi187 ceph-mon[93536]: failed to encode map e476 with expected crc 2024-03-29T05:00:30.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:29 smithi187 ceph-mon[93536]: failed to encode map e476 with expected crc 2024-03-29T05:00:30.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:29 smithi187 ceph-mon[93536]: failed to encode map e476 with expected crc 2024-03-29T05:00:30.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:29 smithi187 ceph-mon[93536]: osdmap e477: 8 total, 8 up, 8 in 2024-03-29T05:00:30.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:29 smithi187 ceph-mon[93536]: failed to encode map e477 with expected crc 2024-03-29T05:00:30.546 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.SnapshotImageState (3932 ms) 2024-03-29T05:00:30.547 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.SnapshotPromoteDemote 2024-03-29T05:00:30.660 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:00:31.078 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:30 smithi181 systemd[1]: Stopping Ceph osd.1 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:00:31.079 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:30 smithi181 ceph-mon[101970]: failed to encode map e477 with expected crc 2024-03-29T05:00:31.079 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:30 smithi181 ceph-mon[101970]: failed to encode map e477 with expected crc 2024-03-29T05:00:31.079 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:30 smithi181 ceph-mon[101970]: failed to encode map e477 with expected crc 2024-03-29T05:00:31.079 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:30 smithi181 ceph-mon[101970]: pgmap v378: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 7.7 KiB/s wr, 136 op/s 2024-03-29T05:00:31.079 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:30 smithi181 ceph-mon[101970]: failed to encode map e478 with expected crc 2024-03-29T05:00:31.079 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:30 smithi181 ceph-mon[101970]: osdmap e478: 8 total, 8 up, 8 in 2024-03-29T05:00:31.079 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:30 smithi181 ceph-mon[106466]: failed to encode map e477 with expected crc 2024-03-29T05:00:31.079 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:30 smithi181 ceph-mon[106466]: failed to encode map e477 with expected crc 2024-03-29T05:00:31.080 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:30 smithi181 ceph-mon[106466]: failed to encode map e477 with expected crc 2024-03-29T05:00:31.080 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:30 smithi181 ceph-mon[106466]: pgmap v378: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 7.7 KiB/s wr, 136 op/s 2024-03-29T05:00:31.080 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:30 smithi181 ceph-mon[106466]: failed to encode map e478 with expected crc 2024-03-29T05:00:31.080 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:30 smithi181 ceph-mon[106466]: osdmap e478: 8 total, 8 up, 8 in 2024-03-29T05:00:31.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:30 smithi187 ceph-mon[93536]: failed to encode map e477 with expected crc 2024-03-29T05:00:31.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:30 smithi187 ceph-mon[93536]: failed to encode map e477 with expected crc 2024-03-29T05:00:31.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:30 smithi187 ceph-mon[93536]: failed to encode map e477 with expected crc 2024-03-29T05:00:31.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:30 smithi187 ceph-mon[93536]: pgmap v378: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 94 KiB/s rd, 7.7 KiB/s wr, 136 op/s 2024-03-29T05:00:31.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:30 smithi187 ceph-mon[93536]: failed to encode map e478 with expected crc 2024-03-29T05:00:31.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:30 smithi187 ceph-mon[93536]: osdmap e478: 8 total, 8 up, 8 in 2024-03-29T05:00:32.176 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:31 smithi181 ceph-mon[106466]: failed to encode map e478 with expected crc 2024-03-29T05:00:32.176 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:31 smithi181 ceph-mon[106466]: failed to encode map e478 with expected crc 2024-03-29T05:00:32.176 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:31 smithi181 ceph-mon[106466]: failed to encode map e478 with expected crc 2024-03-29T05:00:32.176 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:31 smithi181 ceph-mon[106466]: osdmap e479: 8 total, 8 up, 8 in 2024-03-29T05:00:32.176 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:31 smithi181 ceph-mon[106466]: failed to encode map e479 with expected crc 2024-03-29T05:00:32.176 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:31 smithi181 ceph-mon[106466]: osd.1 marked itself down and dead 2024-03-29T05:00:32.176 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1[51865]: 2024-03-29T05:00:31.728+0000 7f9093dd1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T05:00:32.177 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1[51865]: 2024-03-29T05:00:31.728+0000 7f9093dd1700 -1 osd.1 479 *** Got signal Terminated *** 2024-03-29T05:00:32.177 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1[51865]: 2024-03-29T05:00:31.728+0000 7f9093dd1700 -1 osd.1 479 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T05:00:32.177 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:31 smithi181 ceph-mon[101970]: failed to encode map e478 with expected crc 2024-03-29T05:00:32.177 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:31 smithi181 ceph-mon[101970]: failed to encode map e478 with expected crc 2024-03-29T05:00:32.177 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:31 smithi181 ceph-mon[101970]: failed to encode map e478 with expected crc 2024-03-29T05:00:32.177 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:31 smithi181 ceph-mon[101970]: osdmap e479: 8 total, 8 up, 8 in 2024-03-29T05:00:32.177 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:31 smithi181 ceph-mon[101970]: failed to encode map e479 with expected crc 2024-03-29T05:00:32.177 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:31 smithi181 ceph-mon[101970]: osd.1 marked itself down and dead 2024-03-29T05:00:32.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:31 smithi187 ceph-mon[93536]: failed to encode map e478 with expected crc 2024-03-29T05:00:32.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:31 smithi187 ceph-mon[93536]: failed to encode map e478 with expected crc 2024-03-29T05:00:32.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:31 smithi187 ceph-mon[93536]: failed to encode map e478 with expected crc 2024-03-29T05:00:32.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:31 smithi187 ceph-mon[93536]: osdmap e479: 8 total, 8 up, 8 in 2024-03-29T05:00:32.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:31 smithi187 ceph-mon[93536]: failed to encode map e479 with expected crc 2024-03-29T05:00:32.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:31 smithi187 ceph-mon[93536]: osd.1 marked itself down and dead 2024-03-29T05:00:33.204 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:33 smithi181 podman[120795]: 2024-03-29 05:00:33.051428474 +0000 UTC m=+1.449360448 container died 7ed1966c65cf5d47733840c8c75370b01d1822b9f2fd0d4ee4fafcd282ee5d17 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-03-29T05:00:33.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[106466]: failed to encode map e479 with expected crc 2024-03-29T05:00:33.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[106466]: failed to encode map e479 with expected crc 2024-03-29T05:00:33.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[106466]: failed to encode map e479 with expected crc 2024-03-29T05:00:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[106466]: failed to encode map e479 with expected crc 2024-03-29T05:00:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[106466]: pgmap v381: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 2.7 KiB/s wr, 17 op/s 2024-03-29T05:00:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[106466]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T05:00:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[106466]: failed to encode map e480 with expected crc 2024-03-29T05:00:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[106466]: osdmap e480: 8 total, 7 up, 8 in 2024-03-29T05:00:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1560616317' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:33.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[101970]: failed to encode map e479 with expected crc 2024-03-29T05:00:33.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[101970]: failed to encode map e479 with expected crc 2024-03-29T05:00:33.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[101970]: failed to encode map e479 with expected crc 2024-03-29T05:00:33.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[101970]: failed to encode map e479 with expected crc 2024-03-29T05:00:33.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[101970]: pgmap v381: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 2.7 KiB/s wr, 17 op/s 2024-03-29T05:00:33.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[101970]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T05:00:33.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[101970]: failed to encode map e480 with expected crc 2024-03-29T05:00:33.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[101970]: osdmap e480: 8 total, 7 up, 8 in 2024-03-29T05:00:33.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:32 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1560616317' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:33.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:32 smithi187 ceph-mon[93536]: failed to encode map e479 with expected crc 2024-03-29T05:00:33.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:32 smithi187 ceph-mon[93536]: failed to encode map e479 with expected crc 2024-03-29T05:00:33.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:32 smithi187 ceph-mon[93536]: failed to encode map e479 with expected crc 2024-03-29T05:00:33.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:32 smithi187 ceph-mon[93536]: failed to encode map e479 with expected crc 2024-03-29T05:00:33.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:32 smithi187 ceph-mon[93536]: pgmap v381: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 2.7 KiB/s wr, 17 op/s 2024-03-29T05:00:33.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:32 smithi187 ceph-mon[93536]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T05:00:33.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:32 smithi187 ceph-mon[93536]: failed to encode map e480 with expected crc 2024-03-29T05:00:33.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:32 smithi187 ceph-mon[93536]: osdmap e480: 8 total, 7 up, 8 in 2024-03-29T05:00:33.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:32 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1560616317' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:33.704 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:33 smithi181 podman[120795]: 2024-03-29 05:00:33.54240874 +0000 UTC m=+1.940340747 container cleanup 7ed1966c65cf5d47733840c8c75370b01d1822b9f2fd0d4ee4fafcd282ee5d17 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T05:00:33.704 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:33 smithi181 bash[120795]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1 2024-03-29T05:00:34.203 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:33 smithi181 podman[120899]: 2024-03-29 05:00:33.844319328 +0000 UTC m=+0.789297398 container remove 7ed1966c65cf5d47733840c8c75370b01d1822b9f2fd0d4ee4fafcd282ee5d17 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1, org.label-schema.license=GPLv2, RELEASE=quincy-4213e17, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-03-29T05:00:34.454 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:34 smithi181 podman[120993]: 2024-03-29 05:00:34.098592142 +0000 UTC m=+0.017731873 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:00:34.454 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:34 smithi181 podman[120993]: 2024-03-29 05:00:34.203567856 +0000 UTC m=+0.122707612 container create 37382968a3cb8f2f25731a348319c27d463c673336863615d5a678d1bb416841 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux ) 2024-03-29T05:00:34.454 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:34 smithi181 podman[120993]: 2024-03-29 05:00:34.431181482 +0000 UTC m=+0.350321209 container init 37382968a3cb8f2f25731a348319c27d463c673336863615d5a678d1bb416841 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-deactivate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T05:00:34.454 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:34 smithi181 podman[120993]: 2024-03-29 05:00:34.4365123 +0000 UTC m=+0.355652016 container start 37382968a3cb8f2f25731a348319c27d463c673336863615d5a678d1bb416841 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-deactivate, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-03-29T05:00:34.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:34 smithi187 ceph-mon[93536]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:34 smithi187 ceph-mon[93536]: osdmap e481: 8 total, 7 up, 8 in 2024-03-29T05:00:34.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:34 smithi187 ceph-mon[93536]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:34 smithi187 ceph-mon[93536]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:34 smithi187 ceph-mon[93536]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:34 smithi187 ceph-mon[93536]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:34 smithi187 ceph-mon[93536]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:34 smithi187 ceph-mon[93536]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:34 smithi187 ceph-mon[93536]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:34 smithi187 ceph-mon[93536]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:34 smithi187 ceph-mon[93536]: pgmap v384: 105 pgs: 12 stale+active+clean, 93 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 26 KiB/s wr, 242 op/s 2024-03-29T05:00:34.954 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:34 smithi181 podman[120993]: 2024-03-29 05:00:34.556592563 +0000 UTC m=+0.475732289 container attach 37382968a3cb8f2f25731a348319c27d463c673336863615d5a678d1bb416841 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0) 2024-03-29T05:00:34.954 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:34 smithi181 podman[120993]: 2024-03-29 05:00:34.733820275 +0000 UTC m=+0.652959991 container died 37382968a3cb8f2f25731a348319c27d463c673336863615d5a678d1bb416841 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-03-29T05:00:34.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[106466]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[106466]: osdmap e481: 8 total, 7 up, 8 in 2024-03-29T05:00:34.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[106466]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[106466]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[106466]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[106466]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[106466]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[106466]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[106466]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[106466]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[106466]: pgmap v384: 105 pgs: 12 stale+active+clean, 93 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 26 KiB/s wr, 242 op/s 2024-03-29T05:00:34.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[101970]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[101970]: osdmap e481: 8 total, 7 up, 8 in 2024-03-29T05:00:34.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[101970]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[101970]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[101970]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[101970]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[101970]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[101970]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[101970]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[101970]: failed to encode map e481 with expected crc 2024-03-29T05:00:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:34 smithi181 ceph-mon[101970]: pgmap v384: 105 pgs: 12 stale+active+clean, 93 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 26 KiB/s wr, 242 op/s 2024-03-29T05:00:35.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:35 smithi181 ceph-mon[101970]: failed to encode map e482 with expected crc 2024-03-29T05:00:35.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:35 smithi181 ceph-mon[101970]: failed to encode map e482 with expected crc 2024-03-29T05:00:35.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:35 smithi181 ceph-mon[101970]: osdmap e482: 8 total, 7 up, 8 in 2024-03-29T05:00:35.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:35 smithi181 ceph-mon[101970]: failed to encode map e482 with expected crc 2024-03-29T05:00:35.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:35 smithi181 ceph-mon[101970]: failed to encode map e482 with expected crc 2024-03-29T05:00:35.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:35 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/632493573' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:35.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:35 smithi181 ceph-mon[106466]: failed to encode map e482 with expected crc 2024-03-29T05:00:35.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:35 smithi181 ceph-mon[106466]: failed to encode map e482 with expected crc 2024-03-29T05:00:35.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:35 smithi181 ceph-mon[106466]: osdmap e482: 8 total, 7 up, 8 in 2024-03-29T05:00:35.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:35 smithi181 ceph-mon[106466]: failed to encode map e482 with expected crc 2024-03-29T05:00:35.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:35 smithi181 ceph-mon[106466]: failed to encode map e482 with expected crc 2024-03-29T05:00:35.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:35 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/632493573' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:35.705 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:35 smithi181 podman[121039]: 2024-03-29 05:00:35.360771694 +0000 UTC m=+0.622095971 container remove 37382968a3cb8f2f25731a348319c27d463c673336863615d5a678d1bb416841 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-deactivate, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T05:00:35.705 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:35 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.1.service: Deactivated successfully. 2024-03-29T05:00:35.705 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:35 smithi181 systemd[1]: Stopped Ceph osd.1 for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T05:00:35.706 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:35 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.1.service: Consumed 21.765s CPU time. 2024-03-29T05:00:35.706 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:35 smithi181 systemd[1]: Starting Ceph osd.1 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:00:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:35 smithi187 ceph-mon[93536]: failed to encode map e482 with expected crc 2024-03-29T05:00:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:35 smithi187 ceph-mon[93536]: failed to encode map e482 with expected crc 2024-03-29T05:00:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:35 smithi187 ceph-mon[93536]: osdmap e482: 8 total, 7 up, 8 in 2024-03-29T05:00:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:35 smithi187 ceph-mon[93536]: failed to encode map e482 with expected crc 2024-03-29T05:00:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:35 smithi187 ceph-mon[93536]: failed to encode map e482 with expected crc 2024-03-29T05:00:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:35 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/632493573' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:36.081 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:35 smithi181 podman[121172]: 2024-03-29 05:00:35.715136967 +0000 UTC m=+0.017983367 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:00:36.081 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:35 smithi181 podman[121172]: 2024-03-29 05:00:35.84531691 +0000 UTC m=+0.148163314 container create f4f31e057e998a162be615bea053016b9424dc1dda87104f796141115ca2b401 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-activate, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-03-29T05:00:36.454 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 podman[121172]: 2024-03-29 05:00:36.073160822 +0000 UTC m=+0.376007233 container init f4f31e057e998a162be615bea053016b9424dc1dda87104f796141115ca2b401 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326) 2024-03-29T05:00:36.454 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 podman[121172]: 2024-03-29 05:00:36.079137836 +0000 UTC m=+0.381984235 container start f4f31e057e998a162be615bea053016b9424dc1dda87104f796141115ca2b401 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-03-29T05:00:36.454 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 podman[121172]: 2024-03-29 05:00:36.173267423 +0000 UTC m=+0.476113834 container attach f4f31e057e998a162be615bea053016b9424dc1dda87104f796141115ca2b401 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-activate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True) 2024-03-29T05:00:36.825 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-mon[106466]: failed to encode map e483 with expected crc 2024-03-29T05:00:36.826 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-mon[106466]: failed to encode map e483 with expected crc 2024-03-29T05:00:36.826 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-mon[106466]: osdmap e483: 8 total, 7 up, 8 in 2024-03-29T05:00:36.826 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-mon[106466]: failed to encode map e483 with expected crc 2024-03-29T05:00:36.826 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-mon[106466]: failed to encode map e483 with expected crc 2024-03-29T05:00:36.826 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-mon[106466]: failed to encode map e483 with expected crc 2024-03-29T05:00:36.826 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-mon[106466]: pgmap v387: 105 pgs: 12 stale+active+clean, 93 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 162 KiB/s rd, 23 KiB/s wr, 224 op/s 2024-03-29T05:00:36.826 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-mon[101970]: failed to encode map e483 with expected crc 2024-03-29T05:00:36.826 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-mon[101970]: failed to encode map e483 with expected crc 2024-03-29T05:00:36.827 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-mon[101970]: osdmap e483: 8 total, 7 up, 8 in 2024-03-29T05:00:36.827 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-mon[101970]: failed to encode map e483 with expected crc 2024-03-29T05:00:36.827 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-mon[101970]: failed to encode map e483 with expected crc 2024-03-29T05:00:36.827 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-mon[101970]: failed to encode map e483 with expected crc 2024-03-29T05:00:36.827 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-mon[101970]: pgmap v387: 105 pgs: 12 stale+active+clean, 93 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 162 KiB/s rd, 23 KiB/s wr, 224 op/s 2024-03-29T05:00:36.827 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-activate[121187]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-29T05:00:36.827 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-activate[121187]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-29T05:00:36.827 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 bash[121172]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-29T05:00:36.827 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 bash[121172]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-29T05:00:36.827 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 bash[121172]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-29T05:00:36.827 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 bash[121172]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-29T05:00:36.827 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 bash[121172]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-03-29T05:00:36.828 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 bash[121172]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-29T05:00:36.828 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 bash[121172]: --> ceph-volume raw activate successful for osd ID: 1 2024-03-29T05:00:36.828 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-activate[121187]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-29T05:00:36.828 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-activate[121187]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-29T05:00:36.828 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-activate[121187]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-03-29T05:00:36.828 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-activate[121187]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-29T05:00:36.828 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-activate[121187]: --> ceph-volume raw activate successful for osd ID: 1 2024-03-29T05:00:36.828 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:36 smithi181 podman[121172]: 2024-03-29 05:00:36.552519056 +0000 UTC m=+0.855365448 container died f4f31e057e998a162be615bea053016b9424dc1dda87104f796141115ca2b401 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-03-29T05:00:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:36 smithi187 ceph-mon[93536]: failed to encode map e483 with expected crc 2024-03-29T05:00:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:36 smithi187 ceph-mon[93536]: failed to encode map e483 with expected crc 2024-03-29T05:00:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:36 smithi187 ceph-mon[93536]: osdmap e483: 8 total, 7 up, 8 in 2024-03-29T05:00:36.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:36 smithi187 ceph-mon[93536]: failed to encode map e483 with expected crc 2024-03-29T05:00:36.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:36 smithi187 ceph-mon[93536]: failed to encode map e483 with expected crc 2024-03-29T05:00:36.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:36 smithi187 ceph-mon[93536]: failed to encode map e483 with expected crc 2024-03-29T05:00:36.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:36 smithi187 ceph-mon[93536]: pgmap v387: 105 pgs: 12 stale+active+clean, 93 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 162 KiB/s rd, 23 KiB/s wr, 224 op/s 2024-03-29T05:00:37.454 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:37 smithi181 podman[121324]: 2024-03-29 05:00:37.135681466 +0000 UTC m=+0.577861677 container remove f4f31e057e998a162be615bea053016b9424dc1dda87104f796141115ca2b401 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-activate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-03-29T05:00:37.454 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:37 smithi181 podman[121394]: 2024-03-29 05:00:37.252912383 +0000 UTC m=+0.017795342 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:00:37.454 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:37 smithi181 podman[121394]: 2024-03-29 05:00:37.353250255 +0000 UTC m=+0.118133200 container create cb0b10fc92dd2dcc778a1b81e1b7bd9da258c43e906f1c84a7c9ebd72d7c2040 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True) 2024-03-29T05:00:37.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:37 smithi187 ceph-mon[93536]: failed to encode map e484 with expected crc 2024-03-29T05:00:37.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:37 smithi187 ceph-mon[93536]: failed to encode map e484 with expected crc 2024-03-29T05:00:37.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:37 smithi187 ceph-mon[93536]: osdmap e484: 8 total, 7 up, 8 in 2024-03-29T05:00:37.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:37 smithi187 ceph-mon[93536]: failed to encode map e484 with expected crc 2024-03-29T05:00:37.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:37 smithi187 ceph-mon[93536]: failed to encode map e484 with expected crc 2024-03-29T05:00:37.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:37 smithi187 ceph-mon[93536]: osdmap e485: 8 total, 7 up, 8 in 2024-03-29T05:00:37.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:37 smithi181 ceph-mon[106466]: failed to encode map e484 with expected crc 2024-03-29T05:00:37.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:37 smithi181 ceph-mon[106466]: failed to encode map e484 with expected crc 2024-03-29T05:00:37.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:37 smithi181 ceph-mon[106466]: osdmap e484: 8 total, 7 up, 8 in 2024-03-29T05:00:37.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:37 smithi181 ceph-mon[106466]: failed to encode map e484 with expected crc 2024-03-29T05:00:37.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:37 smithi181 ceph-mon[106466]: failed to encode map e484 with expected crc 2024-03-29T05:00:37.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:37 smithi181 ceph-mon[106466]: osdmap e485: 8 total, 7 up, 8 in 2024-03-29T05:00:37.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:37 smithi181 ceph-mon[101970]: failed to encode map e484 with expected crc 2024-03-29T05:00:37.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:37 smithi181 ceph-mon[101970]: failed to encode map e484 with expected crc 2024-03-29T05:00:37.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:37 smithi181 ceph-mon[101970]: osdmap e484: 8 total, 7 up, 8 in 2024-03-29T05:00:37.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:37 smithi181 ceph-mon[101970]: failed to encode map e484 with expected crc 2024-03-29T05:00:37.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:37 smithi181 ceph-mon[101970]: failed to encode map e484 with expected crc 2024-03-29T05:00:37.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:37 smithi181 ceph-mon[101970]: osdmap e485: 8 total, 7 up, 8 in 2024-03-29T05:00:37.955 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:37 smithi181 podman[121394]: 2024-03-29 05:00:37.572825662 +0000 UTC m=+0.337708608 container init cb0b10fc92dd2dcc778a1b81e1b7bd9da258c43e906f1c84a7c9ebd72d7c2040 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T05:00:37.955 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:37 smithi181 podman[121394]: 2024-03-29 05:00:37.57784597 +0000 UTC m=+0.342728916 container start cb0b10fc92dd2dcc778a1b81e1b7bd9da258c43e906f1c84a7c9ebd72d7c2040 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T05:00:37.955 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:37 smithi181 ceph-osd[121458]: -- 172.21.15.181:0/3759314074 <== mon.2 v2:172.21.15.181:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5569f03fb680 con 0x5569f03c5000 2024-03-29T05:00:37.956 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:37 smithi181 bash[121394]: cb0b10fc92dd2dcc778a1b81e1b7bd9da258c43e906f1c84a7c9ebd72d7c2040 2024-03-29T05:00:37.956 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:37 smithi181 systemd[1]: Started Ceph osd.1 for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T05:00:38.454 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1[121432]: 2024-03-29T05:00:38.118+0000 7f61a05097c0 -1 Falling back to public interface 2024-03-29T05:00:38.818 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[106466]: failed to encode map e485 with expected crc 2024-03-29T05:00:38.818 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[106466]: failed to encode map e485 with expected crc 2024-03-29T05:00:38.818 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[106466]: failed to encode map e485 with expected crc 2024-03-29T05:00:38.818 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2801713948' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:38.818 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[106466]: pgmap v390: 105 pgs: 19 active+undersized+degraded, 30 active+undersized, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 103 KiB/s rd, 6.7 KiB/s wr, 136 op/s; 43/327 objects degraded (13.150%) 2024-03-29T05:00:38.818 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:38.818 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:38.818 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:00:38.818 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[106466]: failed to encode map e485 with expected crc 2024-03-29T05:00:38.818 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[106466]: osdmap e486: 8 total, 7 up, 8 in 2024-03-29T05:00:38.819 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[101970]: failed to encode map e485 with expected crc 2024-03-29T05:00:38.819 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[101970]: failed to encode map e485 with expected crc 2024-03-29T05:00:38.819 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[101970]: failed to encode map e485 with expected crc 2024-03-29T05:00:38.819 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2801713948' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:38.819 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[101970]: pgmap v390: 105 pgs: 19 active+undersized+degraded, 30 active+undersized, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 103 KiB/s rd, 6.7 KiB/s wr, 136 op/s; 43/327 objects degraded (13.150%) 2024-03-29T05:00:38.819 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:38.819 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:38.819 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:00:38.819 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[101970]: failed to encode map e485 with expected crc 2024-03-29T05:00:38.819 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-mon[101970]: osdmap e486: 8 total, 7 up, 8 in 2024-03-29T05:00:38.822 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:38 smithi187 ceph-mon[93536]: failed to encode map e485 with expected crc 2024-03-29T05:00:38.822 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:38 smithi187 ceph-mon[93536]: failed to encode map e485 with expected crc 2024-03-29T05:00:38.822 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:38 smithi187 ceph-mon[93536]: failed to encode map e485 with expected crc 2024-03-29T05:00:38.822 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:38 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2801713948' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:38.822 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:38 smithi187 ceph-mon[93536]: pgmap v390: 105 pgs: 19 active+undersized+degraded, 30 active+undersized, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 103 KiB/s rd, 6.7 KiB/s wr, 136 op/s; 43/327 objects degraded (13.150%) 2024-03-29T05:00:38.822 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:38.822 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:38.822 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:00:38.823 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:38 smithi187 ceph-mon[93536]: failed to encode map e485 with expected crc 2024-03-29T05:00:38.823 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:38 smithi187 ceph-mon[93536]: osdmap e486: 8 total, 7 up, 8 in 2024-03-29T05:00:39.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:00:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:00:38] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.43.0" 2024-03-29T05:00:39.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:39 smithi187 ceph-mon[93536]: failed to encode map e486 with expected crc 2024-03-29T05:00:39.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:39 smithi187 ceph-mon[93536]: failed to encode map e486 with expected crc 2024-03-29T05:00:39.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:39 smithi187 ceph-mon[93536]: failed to encode map e486 with expected crc 2024-03-29T05:00:39.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:39 smithi187 ceph-mon[93536]: Health check failed: Degraded data redundancy: 43/327 objects degraded (13.150%), 19 pgs degraded (PG_DEGRADED) 2024-03-29T05:00:39.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:39 smithi187 ceph-mon[93536]: osdmap e487: 8 total, 7 up, 8 in 2024-03-29T05:00:39.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:39 smithi181 ceph-mon[101970]: failed to encode map e486 with expected crc 2024-03-29T05:00:39.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:39 smithi181 ceph-mon[101970]: failed to encode map e486 with expected crc 2024-03-29T05:00:39.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:39 smithi181 ceph-mon[101970]: failed to encode map e486 with expected crc 2024-03-29T05:00:39.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:39 smithi181 ceph-mon[101970]: Health check failed: Degraded data redundancy: 43/327 objects degraded (13.150%), 19 pgs degraded (PG_DEGRADED) 2024-03-29T05:00:39.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:39 smithi181 ceph-mon[101970]: osdmap e487: 8 total, 7 up, 8 in 2024-03-29T05:00:39.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:39 smithi181 ceph-mon[106466]: failed to encode map e486 with expected crc 2024-03-29T05:00:39.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:39 smithi181 ceph-mon[106466]: failed to encode map e486 with expected crc 2024-03-29T05:00:39.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:39 smithi181 ceph-mon[106466]: failed to encode map e486 with expected crc 2024-03-29T05:00:39.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:39 smithi181 ceph-mon[106466]: Health check failed: Degraded data redundancy: 43/327 objects degraded (13.150%), 19 pgs degraded (PG_DEGRADED) 2024-03-29T05:00:39.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:39 smithi181 ceph-mon[106466]: osdmap e487: 8 total, 7 up, 8 in 2024-03-29T05:00:40.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:40 smithi187 ceph-mon[93536]: failed to encode map e487 with expected crc 2024-03-29T05:00:40.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:40 smithi187 ceph-mon[93536]: failed to encode map e487 with expected crc 2024-03-29T05:00:40.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:40 smithi187 ceph-mon[93536]: failed to encode map e487 with expected crc 2024-03-29T05:00:40.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:40 smithi187 ceph-mon[93536]: failed to encode map e487 with expected crc 2024-03-29T05:00:40.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:40 smithi187 ceph-mon[93536]: pgmap v393: 105 pgs: 19 active+undersized+degraded, 30 active+undersized, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 103 KiB/s rd, 6.7 KiB/s wr, 136 op/s; 43/327 objects degraded (13.150%) 2024-03-29T05:00:40.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:40 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3643664218' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:40.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:40 smithi187 ceph-mon[93536]: osdmap e488: 8 total, 7 up, 8 in 2024-03-29T05:00:40.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:40 smithi181 ceph-mon[106466]: failed to encode map e487 with expected crc 2024-03-29T05:00:40.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:40 smithi181 ceph-mon[106466]: failed to encode map e487 with expected crc 2024-03-29T05:00:40.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:40 smithi181 ceph-mon[106466]: failed to encode map e487 with expected crc 2024-03-29T05:00:40.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:40 smithi181 ceph-mon[106466]: failed to encode map e487 with expected crc 2024-03-29T05:00:40.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:40 smithi181 ceph-mon[106466]: pgmap v393: 105 pgs: 19 active+undersized+degraded, 30 active+undersized, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 103 KiB/s rd, 6.7 KiB/s wr, 136 op/s; 43/327 objects degraded (13.150%) 2024-03-29T05:00:40.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:40 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3643664218' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:40.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:40 smithi181 ceph-mon[106466]: osdmap e488: 8 total, 7 up, 8 in 2024-03-29T05:00:40.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:40 smithi181 ceph-mon[101970]: failed to encode map e487 with expected crc 2024-03-29T05:00:40.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:40 smithi181 ceph-mon[101970]: failed to encode map e487 with expected crc 2024-03-29T05:00:40.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:40 smithi181 ceph-mon[101970]: failed to encode map e487 with expected crc 2024-03-29T05:00:40.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:40 smithi181 ceph-mon[101970]: failed to encode map e487 with expected crc 2024-03-29T05:00:40.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:40 smithi181 ceph-mon[101970]: pgmap v393: 105 pgs: 19 active+undersized+degraded, 30 active+undersized, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 103 KiB/s rd, 6.7 KiB/s wr, 136 op/s; 43/327 objects degraded (13.150%) 2024-03-29T05:00:40.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:40 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3643664218' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:40.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:40 smithi181 ceph-mon[101970]: osdmap e488: 8 total, 7 up, 8 in 2024-03-29T05:00:41.280 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:40 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1[121432]: 2024-03-29T05:00:40.987+0000 7f61a05097c0 -1 osd.1 479 log_to_monitors true 2024-03-29T05:00:41.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:41 smithi187 ceph-mon[93536]: failed to encode map e488 with expected crc 2024-03-29T05:00:41.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:41 smithi187 ceph-mon[93536]: failed to encode map e488 with expected crc 2024-03-29T05:00:41.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:41 smithi187 ceph-mon[93536]: failed to encode map e488 with expected crc 2024-03-29T05:00:41.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:41 smithi187 ceph-mon[93536]: failed to encode map e488 with expected crc 2024-03-29T05:00:41.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:41 smithi187 ceph-mon[93536]: failed to encode map e488 with expected crc 2024-03-29T05:00:41.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:41 smithi187 ceph-mon[93536]: from='osd.1 [v2:172.21.15.181:6810/3877140947,v1:172.21.15.181:6811/3877140947]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T05:00:41.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:41.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:00:41.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:41 smithi187 ceph-mon[93536]: from='osd.1 [v2:172.21.15.181:6810/3877140947,v1:172.21.15.181:6811/3877140947]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T05:00:41.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:41 smithi187 ceph-mon[93536]: osdmap e489: 8 total, 7 up, 8 in 2024-03-29T05:00:41.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:41 smithi187 ceph-mon[93536]: from='osd.1 [v2:172.21.15.181:6810/3877140947,v1:172.21.15.181:6811/3877140947]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T05:00:41.902 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.SnapshotPromoteDemote (11355 ms) 2024-03-29T05:00:41.902 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestMirroring.AioSnapshotCreate 2024-03-29T05:00:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[106466]: failed to encode map e488 with expected crc 2024-03-29T05:00:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[106466]: failed to encode map e488 with expected crc 2024-03-29T05:00:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[106466]: failed to encode map e488 with expected crc 2024-03-29T05:00:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[106466]: failed to encode map e488 with expected crc 2024-03-29T05:00:41.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[106466]: failed to encode map e488 with expected crc 2024-03-29T05:00:41.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[106466]: from='osd.1 [v2:172.21.15.181:6810/3877140947,v1:172.21.15.181:6811/3877140947]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T05:00:41.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:41.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:00:41.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[106466]: from='osd.1 [v2:172.21.15.181:6810/3877140947,v1:172.21.15.181:6811/3877140947]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T05:00:41.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[106466]: osdmap e489: 8 total, 7 up, 8 in 2024-03-29T05:00:41.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[106466]: from='osd.1 [v2:172.21.15.181:6810/3877140947,v1:172.21.15.181:6811/3877140947]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T05:00:41.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[101970]: failed to encode map e488 with expected crc 2024-03-29T05:00:41.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[101970]: failed to encode map e488 with expected crc 2024-03-29T05:00:41.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[101970]: failed to encode map e488 with expected crc 2024-03-29T05:00:41.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[101970]: failed to encode map e488 with expected crc 2024-03-29T05:00:41.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[101970]: failed to encode map e488 with expected crc 2024-03-29T05:00:41.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[101970]: from='osd.1 [v2:172.21.15.181:6810/3877140947,v1:172.21.15.181:6811/3877140947]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T05:00:41.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:41.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:00:41.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[101970]: from='osd.1 [v2:172.21.15.181:6810/3877140947,v1:172.21.15.181:6811/3877140947]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T05:00:41.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[101970]: osdmap e489: 8 total, 7 up, 8 in 2024-03-29T05:00:41.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-mon[101970]: from='osd.1 [v2:172.21.15.181:6810/3877140947,v1:172.21.15.181:6811/3877140947]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T05:00:42.454 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:00:41 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1[121432]: 2024-03-29T05:00:41.966+0000 7f618d959700 -1 osd.1 479 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T05:00:42.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:42 smithi187 ceph-mon[93536]: failed to encode map e489 with expected crc 2024-03-29T05:00:42.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:42 smithi187 ceph-mon[93536]: failed to encode map e489 with expected crc 2024-03-29T05:00:42.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:42 smithi187 ceph-mon[93536]: failed to encode map e489 with expected crc 2024-03-29T05:00:42.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:42 smithi187 ceph-mon[93536]: failed to encode map e489 with expected crc 2024-03-29T05:00:42.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:42 smithi187 ceph-mon[93536]: pgmap v396: 105 pgs: 19 active+undersized+degraded, 30 active+undersized, 56 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 511 B/s wr, 3 op/s; 41/318 objects degraded (12.893%) 2024-03-29T05:00:42.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:42 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:42.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:42 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:42.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:42 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:42.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:42 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:42.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:42 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/8d936610-7f30-4b16-b89c-389b7073cfdc"}]: dispatch 2024-03-29T05:00:42.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:42 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/8d936610-7f30-4b16-b89c-389b7073cfdc"}]': finished 2024-03-29T05:00:42.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:42 smithi187 ceph-mon[93536]: from='osd.1 [v2:172.21.15.181:6810/3877140947,v1:172.21.15.181:6811/3877140947]' entity='osd.1' 2024-03-29T05:00:42.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:42 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2911718943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[101970]: failed to encode map e489 with expected crc 2024-03-29T05:00:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[101970]: failed to encode map e489 with expected crc 2024-03-29T05:00:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[101970]: failed to encode map e489 with expected crc 2024-03-29T05:00:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[101970]: failed to encode map e489 with expected crc 2024-03-29T05:00:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[101970]: pgmap v396: 105 pgs: 19 active+undersized+degraded, 30 active+undersized, 56 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 511 B/s wr, 3 op/s; 41/318 objects degraded (12.893%) 2024-03-29T05:00:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:42.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/8d936610-7f30-4b16-b89c-389b7073cfdc"}]: dispatch 2024-03-29T05:00:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/8d936610-7f30-4b16-b89c-389b7073cfdc"}]': finished 2024-03-29T05:00:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[101970]: from='osd.1 [v2:172.21.15.181:6810/3877140947,v1:172.21.15.181:6811/3877140947]' entity='osd.1' 2024-03-29T05:00:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2911718943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:42.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[106466]: failed to encode map e489 with expected crc 2024-03-29T05:00:42.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[106466]: failed to encode map e489 with expected crc 2024-03-29T05:00:42.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[106466]: failed to encode map e489 with expected crc 2024-03-29T05:00:42.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[106466]: failed to encode map e489 with expected crc 2024-03-29T05:00:42.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[106466]: pgmap v396: 105 pgs: 19 active+undersized+degraded, 30 active+undersized, 56 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 511 B/s wr, 3 op/s; 41/318 objects degraded (12.893%) 2024-03-29T05:00:42.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:42.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/8d936610-7f30-4b16-b89c-389b7073cfdc"}]: dispatch 2024-03-29T05:00:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/8d936610-7f30-4b16-b89c-389b7073cfdc"}]': finished 2024-03-29T05:00:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[106466]: from='osd.1 [v2:172.21.15.181:6810/3877140947,v1:172.21.15.181:6811/3877140947]' entity='osd.1' 2024-03-29T05:00:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:42 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2911718943' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:43.861 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:43 smithi181 ceph-mon[101970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T05:00:43.861 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:43 smithi181 ceph-mon[101970]: failed to encode map e490 with expected crc 2024-03-29T05:00:43.861 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:43 smithi181 ceph-mon[101970]: failed to encode map e490 with expected crc 2024-03-29T05:00:43.861 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:43 smithi181 ceph-mon[101970]: osd.1 [v2:172.21.15.181:6810/3877140947,v1:172.21.15.181:6811/3877140947] boot 2024-03-29T05:00:43.861 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:43 smithi181 ceph-mon[101970]: osdmap e490: 8 total, 8 up, 8 in 2024-03-29T05:00:43.861 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:43 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T05:00:43.861 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:43 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:43.861 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:43 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:43.862 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:43 smithi181 ceph-mon[106466]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T05:00:43.862 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:43 smithi181 ceph-mon[106466]: failed to encode map e490 with expected crc 2024-03-29T05:00:43.862 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:43 smithi181 ceph-mon[106466]: failed to encode map e490 with expected crc 2024-03-29T05:00:43.862 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:43 smithi181 ceph-mon[106466]: osd.1 [v2:172.21.15.181:6810/3877140947,v1:172.21.15.181:6811/3877140947] boot 2024-03-29T05:00:43.862 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:43 smithi181 ceph-mon[106466]: osdmap e490: 8 total, 8 up, 8 in 2024-03-29T05:00:43.862 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:43 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T05:00:43.862 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:43 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:43.862 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:43 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:43.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:43 smithi187 ceph-mon[93536]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T05:00:43.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:43 smithi187 ceph-mon[93536]: failed to encode map e490 with expected crc 2024-03-29T05:00:43.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:43 smithi187 ceph-mon[93536]: failed to encode map e490 with expected crc 2024-03-29T05:00:43.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:43 smithi187 ceph-mon[93536]: osd.1 [v2:172.21.15.181:6810/3877140947,v1:172.21.15.181:6811/3877140947] boot 2024-03-29T05:00:43.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:43 smithi187 ceph-mon[93536]: osdmap e490: 8 total, 8 up, 8 in 2024-03-29T05:00:43.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:43 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T05:00:43.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:43 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:43.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:43 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:45.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: pgmap v398: 105 pgs: 18 active+undersized+degraded, 31 active+undersized, 56 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 6.9 KiB/s wr, 55 op/s; 38/306 objects degraded (12.418%) 2024-03-29T05:00:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: osdmap e491: 8 total, 8 up, 8 in 2024-03-29T05:00:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3225406485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:45.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[101970]: Health check update: Degraded data redundancy: 38/306 objects degraded (12.418%), 18 pgs degraded (PG_DEGRADED) 2024-03-29T05:00:45.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: pgmap v398: 105 pgs: 18 active+undersized+degraded, 31 active+undersized, 56 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 6.9 KiB/s wr, 55 op/s; 38/306 objects degraded (12.418%) 2024-03-29T05:00:45.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: osdmap e491: 8 total, 8 up, 8 in 2024-03-29T05:00:45.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3225406485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:45.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:44 smithi181 ceph-mon[106466]: Health check update: Degraded data redundancy: 38/306 objects degraded (12.418%), 18 pgs degraded (PG_DEGRADED) 2024-03-29T05:00:45.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: pgmap v398: 105 pgs: 18 active+undersized+degraded, 31 active+undersized, 56 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 6.9 KiB/s wr, 55 op/s; 38/306 objects degraded (12.418%) 2024-03-29T05:00:45.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: osdmap e491: 8 total, 8 up, 8 in 2024-03-29T05:00:45.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: failed to encode map e491 with expected crc 2024-03-29T05:00:45.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3225406485' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:45.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:44 smithi187 ceph-mon[93536]: Health check update: Degraded data redundancy: 38/306 objects degraded (12.418%), 18 pgs degraded (PG_DEGRADED) 2024-03-29T05:00:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:45 smithi181 ceph-mon[101970]: failed to encode map e492 with expected crc 2024-03-29T05:00:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:45 smithi181 ceph-mon[101970]: failed to encode map e492 with expected crc 2024-03-29T05:00:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:45 smithi181 ceph-mon[101970]: osdmap e492: 8 total, 8 up, 8 in 2024-03-29T05:00:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:45 smithi181 ceph-mon[101970]: failed to encode map e492 with expected crc 2024-03-29T05:00:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:45 smithi181 ceph-mon[101970]: failed to encode map e492 with expected crc 2024-03-29T05:00:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:45 smithi181 ceph-mon[101970]: failed to encode map e492 with expected crc 2024-03-29T05:00:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:45 smithi181 ceph-mon[106466]: failed to encode map e492 with expected crc 2024-03-29T05:00:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:45 smithi181 ceph-mon[106466]: failed to encode map e492 with expected crc 2024-03-29T05:00:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:45 smithi181 ceph-mon[106466]: osdmap e492: 8 total, 8 up, 8 in 2024-03-29T05:00:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:45 smithi181 ceph-mon[106466]: failed to encode map e492 with expected crc 2024-03-29T05:00:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:45 smithi181 ceph-mon[106466]: failed to encode map e492 with expected crc 2024-03-29T05:00:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:45 smithi181 ceph-mon[106466]: failed to encode map e492 with expected crc 2024-03-29T05:00:46.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:45 smithi187 ceph-mon[93536]: failed to encode map e492 with expected crc 2024-03-29T05:00:46.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:45 smithi187 ceph-mon[93536]: failed to encode map e492 with expected crc 2024-03-29T05:00:46.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:45 smithi187 ceph-mon[93536]: osdmap e492: 8 total, 8 up, 8 in 2024-03-29T05:00:46.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:45 smithi187 ceph-mon[93536]: failed to encode map e492 with expected crc 2024-03-29T05:00:46.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:45 smithi187 ceph-mon[93536]: failed to encode map e492 with expected crc 2024-03-29T05:00:46.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:45 smithi187 ceph-mon[93536]: failed to encode map e492 with expected crc 2024-03-29T05:00:46.704 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T05:00:46.450+0000 7f67753a3700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-03-29T05:00:47.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: pgmap v401: 105 pgs: 18 active+undersized+degraded, 31 active+undersized, 56 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 6.4 KiB/s wr, 51 op/s; 38/306 objects degraded (12.418%) 2024-03-29T05:00:47.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: osdmap e493: 8 total, 8 up, 8 in 2024-03-29T05:00:47.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:00:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:00:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:00:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T05:00:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1345346129' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: pgmap v401: 105 pgs: 18 active+undersized+degraded, 31 active+undersized, 56 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 6.4 KiB/s wr, 51 op/s; 38/306 objects degraded (12.418%) 2024-03-29T05:00:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: osdmap e493: 8 total, 8 up, 8 in 2024-03-29T05:00:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:00:47.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:00:47.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:47.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:00:47.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:47.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:47.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:47.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T05:00:47.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:46 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1345346129' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:47.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: pgmap v401: 105 pgs: 18 active+undersized+degraded, 31 active+undersized, 56 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 6.4 KiB/s wr, 51 op/s; 38/306 objects degraded (12.418%) 2024-03-29T05:00:47.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: osdmap e493: 8 total, 8 up, 8 in 2024-03-29T05:00:47.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: failed to encode map e493 with expected crc 2024-03-29T05:00:47.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:47.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:47.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:00:47.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:00:47.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:47.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:00:47.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:47.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:47.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:00:47.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T05:00:47.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:46 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1345346129' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:48.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[101970]: pgmap v403: 105 pgs: 18 active+undersized+degraded, 31 active+undersized, 56 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 38/306 objects degraded (12.418%) 2024-03-29T05:00:48.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[101970]: pgmap v404: 105 pgs: 18 active+undersized+degraded, 31 active+undersized, 56 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 38/306 objects degraded (12.418%) 2024-03-29T05:00:48.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[101970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T05:00:48.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[101970]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-03-29T05:00:48.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[101970]: osdmap e494: 8 total, 8 up, 8 in 2024-03-29T05:00:48.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[101970]: failed to encode map e494 with expected crc 2024-03-29T05:00:48.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[101970]: failed to encode map e494 with expected crc 2024-03-29T05:00:48.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[101970]: failed to encode map e494 with expected crc 2024-03-29T05:00:48.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[101970]: failed to encode map e494 with expected crc 2024-03-29T05:00:48.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[106466]: pgmap v403: 105 pgs: 18 active+undersized+degraded, 31 active+undersized, 56 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 38/306 objects degraded (12.418%) 2024-03-29T05:00:48.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[106466]: pgmap v404: 105 pgs: 18 active+undersized+degraded, 31 active+undersized, 56 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 38/306 objects degraded (12.418%) 2024-03-29T05:00:48.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[106466]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T05:00:48.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[106466]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-03-29T05:00:48.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[106466]: osdmap e494: 8 total, 8 up, 8 in 2024-03-29T05:00:48.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[106466]: failed to encode map e494 with expected crc 2024-03-29T05:00:48.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[106466]: failed to encode map e494 with expected crc 2024-03-29T05:00:48.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[106466]: failed to encode map e494 with expected crc 2024-03-29T05:00:48.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:47 smithi181 ceph-mon[106466]: failed to encode map e494 with expected crc 2024-03-29T05:00:48.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:47 smithi187 ceph-mon[93536]: pgmap v403: 105 pgs: 18 active+undersized+degraded, 31 active+undersized, 56 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 38/306 objects degraded (12.418%) 2024-03-29T05:00:48.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:47 smithi187 ceph-mon[93536]: pgmap v404: 105 pgs: 18 active+undersized+degraded, 31 active+undersized, 56 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 38/306 objects degraded (12.418%) 2024-03-29T05:00:48.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:47 smithi187 ceph-mon[93536]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T05:00:48.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:47 smithi187 ceph-mon[93536]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-03-29T05:00:48.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:47 smithi187 ceph-mon[93536]: osdmap e494: 8 total, 8 up, 8 in 2024-03-29T05:00:48.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:47 smithi187 ceph-mon[93536]: failed to encode map e494 with expected crc 2024-03-29T05:00:48.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:47 smithi187 ceph-mon[93536]: failed to encode map e494 with expected crc 2024-03-29T05:00:48.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:47 smithi187 ceph-mon[93536]: failed to encode map e494 with expected crc 2024-03-29T05:00:48.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:47 smithi187 ceph-mon[93536]: failed to encode map e494 with expected crc 2024-03-29T05:00:49.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:00:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:00:48] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2024-03-29T05:00:49.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:48 smithi181 ceph-mon[106466]: failed to encode map e495 with expected crc 2024-03-29T05:00:49.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:48 smithi181 ceph-mon[106466]: failed to encode map e495 with expected crc 2024-03-29T05:00:49.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:48 smithi181 ceph-mon[106466]: osdmap e495: 8 total, 8 up, 8 in 2024-03-29T05:00:49.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:48 smithi181 ceph-mon[106466]: failed to encode map e495 with expected crc 2024-03-29T05:00:49.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:48 smithi181 ceph-mon[106466]: failed to encode map e495 with expected crc 2024-03-29T05:00:49.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:48 smithi181 ceph-mon[101970]: failed to encode map e495 with expected crc 2024-03-29T05:00:49.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:48 smithi181 ceph-mon[101970]: failed to encode map e495 with expected crc 2024-03-29T05:00:49.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:48 smithi181 ceph-mon[101970]: osdmap e495: 8 total, 8 up, 8 in 2024-03-29T05:00:49.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:48 smithi181 ceph-mon[101970]: failed to encode map e495 with expected crc 2024-03-29T05:00:49.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:48 smithi181 ceph-mon[101970]: failed to encode map e495 with expected crc 2024-03-29T05:00:49.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:48 smithi187 ceph-mon[93536]: failed to encode map e495 with expected crc 2024-03-29T05:00:49.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:48 smithi187 ceph-mon[93536]: failed to encode map e495 with expected crc 2024-03-29T05:00:49.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:48 smithi187 ceph-mon[93536]: osdmap e495: 8 total, 8 up, 8 in 2024-03-29T05:00:49.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:48 smithi187 ceph-mon[93536]: failed to encode map e495 with expected crc 2024-03-29T05:00:49.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:48 smithi187 ceph-mon[93536]: failed to encode map e495 with expected crc 2024-03-29T05:00:50.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:49 smithi181 ceph-mon[106466]: pgmap v407: 105 pgs: 105 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s rd, 17 KiB/s wr, 143 op/s; 13 B/s, 27 keys/s, 3 objects/s recovering 2024-03-29T05:00:50.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:49 smithi181 ceph-mon[106466]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/306 objects degraded (12.418%), 18 pgs degraded) 2024-03-29T05:00:50.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:49 smithi181 ceph-mon[106466]: failed to encode map e496 with expected crc 2024-03-29T05:00:50.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:49 smithi181 ceph-mon[106466]: osdmap e496: 8 total, 8 up, 8 in 2024-03-29T05:00:50.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:49 smithi181 ceph-mon[106466]: failed to encode map e496 with expected crc 2024-03-29T05:00:50.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:49 smithi181 ceph-mon[106466]: failed to encode map e496 with expected crc 2024-03-29T05:00:50.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:49 smithi181 ceph-mon[106466]: failed to encode map e496 with expected crc 2024-03-29T05:00:50.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:49 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2537967658' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:50.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:49 smithi181 ceph-mon[101970]: pgmap v407: 105 pgs: 105 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s rd, 17 KiB/s wr, 143 op/s; 13 B/s, 27 keys/s, 3 objects/s recovering 2024-03-29T05:00:50.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:49 smithi181 ceph-mon[101970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/306 objects degraded (12.418%), 18 pgs degraded) 2024-03-29T05:00:50.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:49 smithi181 ceph-mon[101970]: failed to encode map e496 with expected crc 2024-03-29T05:00:50.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:49 smithi181 ceph-mon[101970]: osdmap e496: 8 total, 8 up, 8 in 2024-03-29T05:00:50.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:49 smithi181 ceph-mon[101970]: failed to encode map e496 with expected crc 2024-03-29T05:00:50.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:49 smithi181 ceph-mon[101970]: failed to encode map e496 with expected crc 2024-03-29T05:00:50.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:49 smithi181 ceph-mon[101970]: failed to encode map e496 with expected crc 2024-03-29T05:00:50.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:49 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2537967658' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:50.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:49 smithi187 ceph-mon[93536]: pgmap v407: 105 pgs: 105 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s rd, 17 KiB/s wr, 143 op/s; 13 B/s, 27 keys/s, 3 objects/s recovering 2024-03-29T05:00:50.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:49 smithi187 ceph-mon[93536]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/306 objects degraded (12.418%), 18 pgs degraded) 2024-03-29T05:00:50.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:49 smithi187 ceph-mon[93536]: failed to encode map e496 with expected crc 2024-03-29T05:00:50.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:49 smithi187 ceph-mon[93536]: osdmap e496: 8 total, 8 up, 8 in 2024-03-29T05:00:50.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:49 smithi187 ceph-mon[93536]: failed to encode map e496 with expected crc 2024-03-29T05:00:50.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:49 smithi187 ceph-mon[93536]: failed to encode map e496 with expected crc 2024-03-29T05:00:50.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:49 smithi187 ceph-mon[93536]: failed to encode map e496 with expected crc 2024-03-29T05:00:50.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:49 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2537967658' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:51.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:50 smithi181 ceph-mon[106466]: failed to encode map e496 with expected crc 2024-03-29T05:00:51.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:50 smithi181 ceph-mon[106466]: failed to encode map e497 with expected crc 2024-03-29T05:00:51.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:50 smithi181 ceph-mon[106466]: osdmap e497: 8 total, 8 up, 8 in 2024-03-29T05:00:51.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:50 smithi181 ceph-mon[106466]: failed to encode map e497 with expected crc 2024-03-29T05:00:51.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:50 smithi181 ceph-mon[106466]: failed to encode map e497 with expected crc 2024-03-29T05:00:51.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:50 smithi181 ceph-mon[106466]: failed to encode map e497 with expected crc 2024-03-29T05:00:51.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:50 smithi181 ceph-mon[101970]: failed to encode map e496 with expected crc 2024-03-29T05:00:51.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:50 smithi181 ceph-mon[101970]: failed to encode map e497 with expected crc 2024-03-29T05:00:51.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:50 smithi181 ceph-mon[101970]: osdmap e497: 8 total, 8 up, 8 in 2024-03-29T05:00:51.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:50 smithi181 ceph-mon[101970]: failed to encode map e497 with expected crc 2024-03-29T05:00:51.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:50 smithi181 ceph-mon[101970]: failed to encode map e497 with expected crc 2024-03-29T05:00:51.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:50 smithi181 ceph-mon[101970]: failed to encode map e497 with expected crc 2024-03-29T05:00:51.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:50 smithi187 ceph-mon[93536]: failed to encode map e496 with expected crc 2024-03-29T05:00:51.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:50 smithi187 ceph-mon[93536]: failed to encode map e497 with expected crc 2024-03-29T05:00:51.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:50 smithi187 ceph-mon[93536]: osdmap e497: 8 total, 8 up, 8 in 2024-03-29T05:00:51.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:50 smithi187 ceph-mon[93536]: failed to encode map e497 with expected crc 2024-03-29T05:00:51.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:50 smithi187 ceph-mon[93536]: failed to encode map e497 with expected crc 2024-03-29T05:00:51.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:50 smithi187 ceph-mon[93536]: failed to encode map e497 with expected crc 2024-03-29T05:00:52.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:51 smithi181 ceph-mon[106466]: pgmap v410: 105 pgs: 105 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 11 KiB/s wr, 92 op/s; 8 B/s, 17 keys/s, 2 objects/s recovering 2024-03-29T05:00:52.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:51 smithi181 ceph-mon[106466]: failed to encode map e498 with expected crc 2024-03-29T05:00:52.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:51 smithi181 ceph-mon[106466]: failed to encode map e498 with expected crc 2024-03-29T05:00:52.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:51 smithi181 ceph-mon[106466]: osdmap e498: 8 total, 8 up, 8 in 2024-03-29T05:00:52.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:51 smithi181 ceph-mon[106466]: failed to encode map e498 with expected crc 2024-03-29T05:00:52.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:51 smithi181 ceph-mon[106466]: failed to encode map e498 with expected crc 2024-03-29T05:00:52.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:51 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/46655329' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:52.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:51 smithi181 ceph-mon[101970]: pgmap v410: 105 pgs: 105 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 11 KiB/s wr, 92 op/s; 8 B/s, 17 keys/s, 2 objects/s recovering 2024-03-29T05:00:52.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:51 smithi181 ceph-mon[101970]: failed to encode map e498 with expected crc 2024-03-29T05:00:52.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:51 smithi181 ceph-mon[101970]: failed to encode map e498 with expected crc 2024-03-29T05:00:52.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:51 smithi181 ceph-mon[101970]: osdmap e498: 8 total, 8 up, 8 in 2024-03-29T05:00:52.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:51 smithi181 ceph-mon[101970]: failed to encode map e498 with expected crc 2024-03-29T05:00:52.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:51 smithi181 ceph-mon[101970]: failed to encode map e498 with expected crc 2024-03-29T05:00:52.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:51 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/46655329' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:52.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:51 smithi187 ceph-mon[93536]: pgmap v410: 105 pgs: 105 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 11 KiB/s wr, 92 op/s; 8 B/s, 17 keys/s, 2 objects/s recovering 2024-03-29T05:00:52.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:51 smithi187 ceph-mon[93536]: failed to encode map e498 with expected crc 2024-03-29T05:00:52.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:51 smithi187 ceph-mon[93536]: failed to encode map e498 with expected crc 2024-03-29T05:00:52.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:51 smithi187 ceph-mon[93536]: osdmap e498: 8 total, 8 up, 8 in 2024-03-29T05:00:52.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:51 smithi187 ceph-mon[93536]: failed to encode map e498 with expected crc 2024-03-29T05:00:52.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:51 smithi187 ceph-mon[93536]: failed to encode map e498 with expected crc 2024-03-29T05:00:52.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:51 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/46655329' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:53.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:52 smithi187 ceph-mon[93536]: failed to encode map e499 with expected crc 2024-03-29T05:00:53.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:52 smithi187 ceph-mon[93536]: failed to encode map e499 with expected crc 2024-03-29T05:00:53.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:52 smithi187 ceph-mon[93536]: osdmap e499: 8 total, 8 up, 8 in 2024-03-29T05:00:53.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:52 smithi187 ceph-mon[93536]: failed to encode map e499 with expected crc 2024-03-29T05:00:53.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:52 smithi187 ceph-mon[93536]: failed to encode map e499 with expected crc 2024-03-29T05:00:53.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:52 smithi187 ceph-mon[93536]: failed to encode map e499 with expected crc 2024-03-29T05:00:53.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:52 smithi187 ceph-mon[93536]: failed to encode map e500 with expected crc 2024-03-29T05:00:53.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:52 smithi187 ceph-mon[93536]: osdmap e500: 8 total, 8 up, 8 in 2024-03-29T05:00:53.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:52 smithi187 ceph-mon[93536]: failed to encode map e500 with expected crc 2024-03-29T05:00:53.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[106466]: failed to encode map e499 with expected crc 2024-03-29T05:00:53.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[106466]: failed to encode map e499 with expected crc 2024-03-29T05:00:53.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[106466]: osdmap e499: 8 total, 8 up, 8 in 2024-03-29T05:00:53.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[106466]: failed to encode map e499 with expected crc 2024-03-29T05:00:53.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[106466]: failed to encode map e499 with expected crc 2024-03-29T05:00:53.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[106466]: failed to encode map e499 with expected crc 2024-03-29T05:00:53.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[106466]: failed to encode map e500 with expected crc 2024-03-29T05:00:53.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[106466]: osdmap e500: 8 total, 8 up, 8 in 2024-03-29T05:00:53.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[106466]: failed to encode map e500 with expected crc 2024-03-29T05:00:53.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[101970]: failed to encode map e499 with expected crc 2024-03-29T05:00:53.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[101970]: failed to encode map e499 with expected crc 2024-03-29T05:00:53.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[101970]: osdmap e499: 8 total, 8 up, 8 in 2024-03-29T05:00:53.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[101970]: failed to encode map e499 with expected crc 2024-03-29T05:00:53.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[101970]: failed to encode map e499 with expected crc 2024-03-29T05:00:53.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[101970]: failed to encode map e499 with expected crc 2024-03-29T05:00:53.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[101970]: failed to encode map e500 with expected crc 2024-03-29T05:00:53.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[101970]: osdmap e500: 8 total, 8 up, 8 in 2024-03-29T05:00:53.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:52 smithi181 ceph-mon[101970]: failed to encode map e500 with expected crc 2024-03-29T05:00:54.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:53 smithi187 ceph-mon[93536]: pgmap v413: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 76 KiB/s rd, 13 KiB/s wr, 111 op/s 2024-03-29T05:00:54.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:53 smithi187 ceph-mon[93536]: failed to encode map e500 with expected crc 2024-03-29T05:00:54.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:53 smithi187 ceph-mon[93536]: failed to encode map e501 with expected crc 2024-03-29T05:00:54.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:53 smithi187 ceph-mon[93536]: osdmap e501: 8 total, 8 up, 8 in 2024-03-29T05:00:54.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:53 smithi187 ceph-mon[93536]: failed to encode map e501 with expected crc 2024-03-29T05:00:54.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:53 smithi187 ceph-mon[93536]: failed to encode map e501 with expected crc 2024-03-29T05:00:54.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:53 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2461878931' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:54.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:53 smithi181 ceph-mon[101970]: pgmap v413: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 76 KiB/s rd, 13 KiB/s wr, 111 op/s 2024-03-29T05:00:54.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:53 smithi181 ceph-mon[101970]: failed to encode map e500 with expected crc 2024-03-29T05:00:54.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:53 smithi181 ceph-mon[101970]: failed to encode map e501 with expected crc 2024-03-29T05:00:54.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:53 smithi181 ceph-mon[101970]: osdmap e501: 8 total, 8 up, 8 in 2024-03-29T05:00:54.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:53 smithi181 ceph-mon[101970]: failed to encode map e501 with expected crc 2024-03-29T05:00:54.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:53 smithi181 ceph-mon[101970]: failed to encode map e501 with expected crc 2024-03-29T05:00:54.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:53 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2461878931' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:54.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:53 smithi181 ceph-mon[106466]: pgmap v413: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 76 KiB/s rd, 13 KiB/s wr, 111 op/s 2024-03-29T05:00:54.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:53 smithi181 ceph-mon[106466]: failed to encode map e500 with expected crc 2024-03-29T05:00:54.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:53 smithi181 ceph-mon[106466]: failed to encode map e501 with expected crc 2024-03-29T05:00:54.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:53 smithi181 ceph-mon[106466]: osdmap e501: 8 total, 8 up, 8 in 2024-03-29T05:00:54.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:53 smithi181 ceph-mon[106466]: failed to encode map e501 with expected crc 2024-03-29T05:00:54.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:53 smithi181 ceph-mon[106466]: failed to encode map e501 with expected crc 2024-03-29T05:00:54.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:53 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2461878931' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:55.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:54 smithi187 ceph-mon[93536]: failed to encode map e501 with expected crc 2024-03-29T05:00:55.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:54 smithi187 ceph-mon[93536]: failed to encode map e501 with expected crc 2024-03-29T05:00:55.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:54 smithi187 ceph-mon[93536]: failed to encode map e501 with expected crc 2024-03-29T05:00:55.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:54 smithi187 ceph-mon[93536]: osdmap e502: 8 total, 8 up, 8 in 2024-03-29T05:00:55.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:54 smithi181 ceph-mon[101970]: failed to encode map e501 with expected crc 2024-03-29T05:00:55.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:54 smithi181 ceph-mon[101970]: failed to encode map e501 with expected crc 2024-03-29T05:00:55.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:54 smithi181 ceph-mon[101970]: failed to encode map e501 with expected crc 2024-03-29T05:00:55.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:54 smithi181 ceph-mon[101970]: osdmap e502: 8 total, 8 up, 8 in 2024-03-29T05:00:55.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:54 smithi181 ceph-mon[106466]: failed to encode map e501 with expected crc 2024-03-29T05:00:55.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:54 smithi181 ceph-mon[106466]: failed to encode map e501 with expected crc 2024-03-29T05:00:55.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:54 smithi181 ceph-mon[106466]: failed to encode map e501 with expected crc 2024-03-29T05:00:55.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:54 smithi181 ceph-mon[106466]: osdmap e502: 8 total, 8 up, 8 in 2024-03-29T05:00:56.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:55 smithi187 ceph-mon[93536]: pgmap v416: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 76 KiB/s rd, 13 KiB/s wr, 111 op/s 2024-03-29T05:00:56.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:55 smithi187 ceph-mon[93536]: failed to encode map e502 with expected crc 2024-03-29T05:00:56.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:55 smithi187 ceph-mon[93536]: failed to encode map e502 with expected crc 2024-03-29T05:00:56.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:55 smithi187 ceph-mon[93536]: osdmap e503: 8 total, 8 up, 8 in 2024-03-29T05:00:56.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:55 smithi187 ceph-mon[93536]: failed to encode map e503 with expected crc 2024-03-29T05:00:56.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:55 smithi181 ceph-mon[101970]: pgmap v416: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 76 KiB/s rd, 13 KiB/s wr, 111 op/s 2024-03-29T05:00:56.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:55 smithi181 ceph-mon[101970]: failed to encode map e502 with expected crc 2024-03-29T05:00:56.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:55 smithi181 ceph-mon[101970]: failed to encode map e502 with expected crc 2024-03-29T05:00:56.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:55 smithi181 ceph-mon[101970]: osdmap e503: 8 total, 8 up, 8 in 2024-03-29T05:00:56.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:55 smithi181 ceph-mon[101970]: failed to encode map e503 with expected crc 2024-03-29T05:00:56.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:55 smithi181 ceph-mon[106466]: pgmap v416: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 76 KiB/s rd, 13 KiB/s wr, 111 op/s 2024-03-29T05:00:56.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:55 smithi181 ceph-mon[106466]: failed to encode map e502 with expected crc 2024-03-29T05:00:56.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:55 smithi181 ceph-mon[106466]: failed to encode map e502 with expected crc 2024-03-29T05:00:56.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:55 smithi181 ceph-mon[106466]: osdmap e503: 8 total, 8 up, 8 in 2024-03-29T05:00:56.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:55 smithi181 ceph-mon[106466]: failed to encode map e503 with expected crc 2024-03-29T05:00:57.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:56 smithi187 ceph-mon[93536]: failed to encode map e503 with expected crc 2024-03-29T05:00:57.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:56 smithi187 ceph-mon[93536]: failed to encode map e503 with expected crc 2024-03-29T05:00:57.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:56 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2179810652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:57.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:57.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:00:57.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:57.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:56 smithi187 ceph-mon[93536]: osdmap e504: 8 total, 8 up, 8 in 2024-03-29T05:00:57.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:56 smithi187 ceph-mon[93536]: failed to encode map e504 with expected crc 2024-03-29T05:00:57.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:56 smithi187 ceph-mon[93536]: failed to encode map e504 with expected crc 2024-03-29T05:00:57.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[106466]: failed to encode map e503 with expected crc 2024-03-29T05:00:57.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[106466]: failed to encode map e503 with expected crc 2024-03-29T05:00:57.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2179810652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:57.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:57.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:00:57.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:57.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[106466]: osdmap e504: 8 total, 8 up, 8 in 2024-03-29T05:00:57.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[106466]: failed to encode map e504 with expected crc 2024-03-29T05:00:57.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[106466]: failed to encode map e504 with expected crc 2024-03-29T05:00:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[101970]: failed to encode map e503 with expected crc 2024-03-29T05:00:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[101970]: failed to encode map e503 with expected crc 2024-03-29T05:00:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2179810652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:00:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:00:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[101970]: osdmap e504: 8 total, 8 up, 8 in 2024-03-29T05:00:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[101970]: failed to encode map e504 with expected crc 2024-03-29T05:00:57.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:56 smithi181 ceph-mon[101970]: failed to encode map e504 with expected crc 2024-03-29T05:00:58.286 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:57 smithi187 ceph-mon[93536]: pgmap v419: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:00:58.286 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:57 smithi187 ceph-mon[93536]: failed to encode map e504 with expected crc 2024-03-29T05:00:58.286 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:57 smithi187 ceph-mon[93536]: failed to encode map e504 with expected crc 2024-03-29T05:00:58.286 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:57 smithi187 ceph-mon[93536]: osdmap e505: 8 total, 8 up, 8 in 2024-03-29T05:00:58.286 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:57 smithi187 ceph-mon[93536]: failed to encode map e505 with expected crc 2024-03-29T05:00:58.286 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:57 smithi187 ceph-mon[93536]: failed to encode map e505 with expected crc 2024-03-29T05:00:58.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:57 smithi181 ceph-mon[101970]: pgmap v419: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:00:58.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:57 smithi181 ceph-mon[101970]: failed to encode map e504 with expected crc 2024-03-29T05:00:58.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:57 smithi181 ceph-mon[101970]: failed to encode map e504 with expected crc 2024-03-29T05:00:58.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:57 smithi181 ceph-mon[101970]: osdmap e505: 8 total, 8 up, 8 in 2024-03-29T05:00:58.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:57 smithi181 ceph-mon[101970]: failed to encode map e505 with expected crc 2024-03-29T05:00:58.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:57 smithi181 ceph-mon[101970]: failed to encode map e505 with expected crc 2024-03-29T05:00:58.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:57 smithi181 ceph-mon[106466]: pgmap v419: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:00:58.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:57 smithi181 ceph-mon[106466]: failed to encode map e504 with expected crc 2024-03-29T05:00:58.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:57 smithi181 ceph-mon[106466]: failed to encode map e504 with expected crc 2024-03-29T05:00:58.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:57 smithi181 ceph-mon[106466]: osdmap e505: 8 total, 8 up, 8 in 2024-03-29T05:00:58.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:57 smithi181 ceph-mon[106466]: failed to encode map e505 with expected crc 2024-03-29T05:00:58.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:57 smithi181 ceph-mon[106466]: failed to encode map e505 with expected crc 2024-03-29T05:00:59.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:00:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:00:58] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2024-03-29T05:00:59.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:58 smithi181 ceph-mon[106466]: failed to encode map e505 with expected crc 2024-03-29T05:00:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:58 smithi181 ceph-mon[106466]: failed to encode map e505 with expected crc 2024-03-29T05:00:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:58 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3520827016' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:58 smithi181 ceph-mon[106466]: osdmap e506: 8 total, 8 up, 8 in 2024-03-29T05:00:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:58 smithi181 ceph-mon[106466]: failed to encode map e506 with expected crc 2024-03-29T05:00:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:58 smithi181 ceph-mon[106466]: failed to encode map e506 with expected crc 2024-03-29T05:00:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:58 smithi181 ceph-mon[101970]: failed to encode map e505 with expected crc 2024-03-29T05:00:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:58 smithi181 ceph-mon[101970]: failed to encode map e505 with expected crc 2024-03-29T05:00:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:58 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3520827016' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:58 smithi181 ceph-mon[101970]: osdmap e506: 8 total, 8 up, 8 in 2024-03-29T05:00:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:58 smithi181 ceph-mon[101970]: failed to encode map e506 with expected crc 2024-03-29T05:00:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:58 smithi181 ceph-mon[101970]: failed to encode map e506 with expected crc 2024-03-29T05:00:59.302 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:58 smithi187 ceph-mon[93536]: failed to encode map e505 with expected crc 2024-03-29T05:00:59.303 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:58 smithi187 ceph-mon[93536]: failed to encode map e505 with expected crc 2024-03-29T05:00:59.303 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:58 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3520827016' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:00:59.303 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:58 smithi187 ceph-mon[93536]: osdmap e506: 8 total, 8 up, 8 in 2024-03-29T05:00:59.303 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:58 smithi187 ceph-mon[93536]: failed to encode map e506 with expected crc 2024-03-29T05:00:59.303 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:58 smithi187 ceph-mon[93536]: failed to encode map e506 with expected crc 2024-03-29T05:01:00.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:59 smithi187 ceph-mon[93536]: pgmap v422: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 21 KiB/s wr, 244 op/s 2024-03-29T05:01:00.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:59 smithi187 ceph-mon[93536]: failed to encode map e506 with expected crc 2024-03-29T05:01:00.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:59 smithi187 ceph-mon[93536]: failed to encode map e506 with expected crc 2024-03-29T05:01:00.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:59 smithi187 ceph-mon[93536]: failed to encode map e507 with expected crc 2024-03-29T05:01:00.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:59 smithi187 ceph-mon[93536]: osdmap e507: 8 total, 8 up, 8 in 2024-03-29T05:01:00.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:59 smithi187 ceph-mon[93536]: failed to encode map e507 with expected crc 2024-03-29T05:01:00.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:00:59 smithi187 ceph-mon[93536]: failed to encode map e507 with expected crc 2024-03-29T05:01:00.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:59 smithi181 ceph-mon[106466]: pgmap v422: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 21 KiB/s wr, 244 op/s 2024-03-29T05:01:00.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:59 smithi181 ceph-mon[106466]: failed to encode map e506 with expected crc 2024-03-29T05:01:00.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:59 smithi181 ceph-mon[106466]: failed to encode map e506 with expected crc 2024-03-29T05:01:00.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:59 smithi181 ceph-mon[106466]: failed to encode map e507 with expected crc 2024-03-29T05:01:00.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:59 smithi181 ceph-mon[106466]: osdmap e507: 8 total, 8 up, 8 in 2024-03-29T05:01:00.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:59 smithi181 ceph-mon[106466]: failed to encode map e507 with expected crc 2024-03-29T05:01:00.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:00:59 smithi181 ceph-mon[106466]: failed to encode map e507 with expected crc 2024-03-29T05:01:00.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:59 smithi181 ceph-mon[101970]: pgmap v422: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 21 KiB/s wr, 244 op/s 2024-03-29T05:01:00.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:59 smithi181 ceph-mon[101970]: failed to encode map e506 with expected crc 2024-03-29T05:01:00.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:59 smithi181 ceph-mon[101970]: failed to encode map e506 with expected crc 2024-03-29T05:01:00.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:59 smithi181 ceph-mon[101970]: failed to encode map e507 with expected crc 2024-03-29T05:01:00.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:59 smithi181 ceph-mon[101970]: osdmap e507: 8 total, 8 up, 8 in 2024-03-29T05:01:00.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:59 smithi181 ceph-mon[101970]: failed to encode map e507 with expected crc 2024-03-29T05:01:00.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:00:59 smithi181 ceph-mon[101970]: failed to encode map e507 with expected crc 2024-03-29T05:01:01.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:00 smithi187 ceph-mon[93536]: failed to encode map e507 with expected crc 2024-03-29T05:01:01.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:00 smithi187 ceph-mon[93536]: failed to encode map e507 with expected crc 2024-03-29T05:01:01.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:00 smithi187 ceph-mon[93536]: pgmap v425: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 21 KiB/s wr, 244 op/s 2024-03-29T05:01:01.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:00 smithi187 ceph-mon[93536]: osdmap e508: 8 total, 8 up, 8 in 2024-03-29T05:01:01.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:00 smithi187 ceph-mon[93536]: failed to encode map e508 with expected crc 2024-03-29T05:01:01.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:00 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1022776498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:01.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:00 smithi181 ceph-mon[101970]: failed to encode map e507 with expected crc 2024-03-29T05:01:01.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:00 smithi181 ceph-mon[101970]: failed to encode map e507 with expected crc 2024-03-29T05:01:01.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:00 smithi181 ceph-mon[101970]: pgmap v425: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 21 KiB/s wr, 244 op/s 2024-03-29T05:01:01.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:00 smithi181 ceph-mon[101970]: osdmap e508: 8 total, 8 up, 8 in 2024-03-29T05:01:01.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:00 smithi181 ceph-mon[101970]: failed to encode map e508 with expected crc 2024-03-29T05:01:01.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:00 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1022776498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:01.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:00 smithi181 ceph-mon[106466]: failed to encode map e507 with expected crc 2024-03-29T05:01:01.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:00 smithi181 ceph-mon[106466]: failed to encode map e507 with expected crc 2024-03-29T05:01:01.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:00 smithi181 ceph-mon[106466]: pgmap v425: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 181 KiB/s rd, 21 KiB/s wr, 244 op/s 2024-03-29T05:01:01.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:00 smithi181 ceph-mon[106466]: osdmap e508: 8 total, 8 up, 8 in 2024-03-29T05:01:01.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:00 smithi181 ceph-mon[106466]: failed to encode map e508 with expected crc 2024-03-29T05:01:01.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:00 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1022776498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:02.242 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:01 smithi181 ceph-mon[106466]: failed to encode map e508 with expected crc 2024-03-29T05:01:02.242 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:01 smithi181 ceph-mon[106466]: failed to encode map e508 with expected crc 2024-03-29T05:01:02.242 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:01 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T05:01:02.242 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:01 smithi181 ceph-mon[106466]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T05:01:02.242 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:01 smithi181 ceph-mon[106466]: Upgrade: osd.2 is safe to restart 2024-03-29T05:01:02.242 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:01 smithi181 ceph-mon[106466]: osdmap e509: 8 total, 8 up, 8 in 2024-03-29T05:01:02.242 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:01 smithi181 ceph-mon[106466]: failed to encode map e509 with expected crc 2024-03-29T05:01:02.243 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:01 smithi181 ceph-mon[106466]: failed to encode map e509 with expected crc 2024-03-29T05:01:02.243 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:01 smithi181 ceph-mon[101970]: failed to encode map e508 with expected crc 2024-03-29T05:01:02.243 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:01 smithi181 ceph-mon[101970]: failed to encode map e508 with expected crc 2024-03-29T05:01:02.243 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:01 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T05:01:02.243 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:01 smithi181 ceph-mon[101970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T05:01:02.243 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:01 smithi181 ceph-mon[101970]: Upgrade: osd.2 is safe to restart 2024-03-29T05:01:02.243 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:01 smithi181 ceph-mon[101970]: osdmap e509: 8 total, 8 up, 8 in 2024-03-29T05:01:02.243 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:01 smithi181 ceph-mon[101970]: failed to encode map e509 with expected crc 2024-03-29T05:01:02.243 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:01 smithi181 ceph-mon[101970]: failed to encode map e509 with expected crc 2024-03-29T05:01:02.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:01 smithi187 ceph-mon[93536]: failed to encode map e508 with expected crc 2024-03-29T05:01:02.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:01 smithi187 ceph-mon[93536]: failed to encode map e508 with expected crc 2024-03-29T05:01:02.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:01 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T05:01:02.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:01 smithi187 ceph-mon[93536]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T05:01:02.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:01 smithi187 ceph-mon[93536]: Upgrade: osd.2 is safe to restart 2024-03-29T05:01:02.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:01 smithi187 ceph-mon[93536]: osdmap e509: 8 total, 8 up, 8 in 2024-03-29T05:01:02.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:01 smithi187 ceph-mon[93536]: failed to encode map e509 with expected crc 2024-03-29T05:01:02.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:01 smithi187 ceph-mon[93536]: failed to encode map e509 with expected crc 2024-03-29T05:01:03.149 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:02 smithi181 ceph-mon[101970]: failed to encode map e509 with expected crc 2024-03-29T05:01:03.149 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:02 smithi181 ceph-mon[101970]: failed to encode map e509 with expected crc 2024-03-29T05:01:03.149 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:02 smithi181 ceph-mon[101970]: failed to encode map e509 with expected crc 2024-03-29T05:01:03.149 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:02 smithi181 ceph-mon[101970]: pgmap v428: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 8.7 KiB/s wr, 158 op/s 2024-03-29T05:01:03.149 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:02 smithi181 ceph-mon[101970]: osdmap e510: 8 total, 8 up, 8 in 2024-03-29T05:01:03.149 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:02 smithi181 ceph-mon[101970]: failed to encode map e510 with expected crc 2024-03-29T05:01:03.149 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:02 smithi181 ceph-mon[101970]: failed to encode map e510 with expected crc 2024-03-29T05:01:03.149 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:02 smithi181 ceph-mon[106466]: failed to encode map e509 with expected crc 2024-03-29T05:01:03.150 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:02 smithi181 ceph-mon[106466]: failed to encode map e509 with expected crc 2024-03-29T05:01:03.150 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:02 smithi181 ceph-mon[106466]: failed to encode map e509 with expected crc 2024-03-29T05:01:03.150 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:02 smithi181 ceph-mon[106466]: pgmap v428: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 8.7 KiB/s wr, 158 op/s 2024-03-29T05:01:03.150 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:02 smithi181 ceph-mon[106466]: osdmap e510: 8 total, 8 up, 8 in 2024-03-29T05:01:03.150 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:02 smithi181 ceph-mon[106466]: failed to encode map e510 with expected crc 2024-03-29T05:01:03.150 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:02 smithi181 ceph-mon[106466]: failed to encode map e510 with expected crc 2024-03-29T05:01:03.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:02 smithi187 ceph-mon[93536]: failed to encode map e509 with expected crc 2024-03-29T05:01:03.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:02 smithi187 ceph-mon[93536]: failed to encode map e509 with expected crc 2024-03-29T05:01:03.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:02 smithi187 ceph-mon[93536]: failed to encode map e509 with expected crc 2024-03-29T05:01:03.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:02 smithi187 ceph-mon[93536]: pgmap v428: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 8.7 KiB/s wr, 158 op/s 2024-03-29T05:01:03.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:02 smithi187 ceph-mon[93536]: osdmap e510: 8 total, 8 up, 8 in 2024-03-29T05:01:03.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:02 smithi187 ceph-mon[93536]: failed to encode map e510 with expected crc 2024-03-29T05:01:03.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:02 smithi187 ceph-mon[93536]: failed to encode map e510 with expected crc 2024-03-29T05:01:04.043 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[101970]: failed to encode map e510 with expected crc 2024-03-29T05:01:04.044 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[101970]: failed to encode map e510 with expected crc 2024-03-29T05:01:04.044 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[101970]: failed to encode map e510 with expected crc 2024-03-29T05:01:04.044 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/297020676' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:04.044 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[101970]: Upgrade: Updating osd.2 2024-03-29T05:01:04.044 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:04.044 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T05:01:04.044 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:01:04.044 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[101970]: Deploying daemon osd.2 on smithi181 2024-03-29T05:01:04.044 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[101970]: osdmap e511: 8 total, 8 up, 8 in 2024-03-29T05:01:04.044 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[101970]: failed to encode map e511 with expected crc 2024-03-29T05:01:04.044 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[101970]: failed to encode map e511 with expected crc 2024-03-29T05:01:04.045 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[106466]: failed to encode map e510 with expected crc 2024-03-29T05:01:04.045 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[106466]: failed to encode map e510 with expected crc 2024-03-29T05:01:04.045 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[106466]: failed to encode map e510 with expected crc 2024-03-29T05:01:04.045 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/297020676' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:04.045 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[106466]: Upgrade: Updating osd.2 2024-03-29T05:01:04.045 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:04.045 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T05:01:04.045 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:01:04.045 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[106466]: Deploying daemon osd.2 on smithi181 2024-03-29T05:01:04.045 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[106466]: osdmap e511: 8 total, 8 up, 8 in 2024-03-29T05:01:04.045 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[106466]: failed to encode map e511 with expected crc 2024-03-29T05:01:04.045 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:03 smithi181 ceph-mon[106466]: failed to encode map e511 with expected crc 2024-03-29T05:01:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:03 smithi187 ceph-mon[93536]: failed to encode map e510 with expected crc 2024-03-29T05:01:04.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:03 smithi187 ceph-mon[93536]: failed to encode map e510 with expected crc 2024-03-29T05:01:04.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:03 smithi187 ceph-mon[93536]: failed to encode map e510 with expected crc 2024-03-29T05:01:04.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:03 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/297020676' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:04.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:03 smithi187 ceph-mon[93536]: Upgrade: Updating osd.2 2024-03-29T05:01:04.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:03 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:04.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:03 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T05:01:04.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:03 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:01:04.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:03 smithi187 ceph-mon[93536]: Deploying daemon osd.2 on smithi181 2024-03-29T05:01:04.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:03 smithi187 ceph-mon[93536]: osdmap e511: 8 total, 8 up, 8 in 2024-03-29T05:01:04.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:03 smithi187 ceph-mon[93536]: failed to encode map e511 with expected crc 2024-03-29T05:01:04.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:03 smithi187 ceph-mon[93536]: failed to encode map e511 with expected crc 2024-03-29T05:01:05.360 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[101970]: failed to encode map e511 with expected crc 2024-03-29T05:01:05.360 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[101970]: failed to encode map e511 with expected crc 2024-03-29T05:01:05.360 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[101970]: failed to encode map e511 with expected crc 2024-03-29T05:01:05.360 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[101970]: pgmap v431: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 8.7 KiB/s wr, 158 op/s 2024-03-29T05:01:05.360 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[101970]: failed to encode map e512 with expected crc 2024-03-29T05:01:05.360 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[101970]: osdmap e512: 8 total, 8 up, 8 in 2024-03-29T05:01:05.361 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[101970]: failed to encode map e512 with expected crc 2024-03-29T05:01:05.361 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[101970]: failed to encode map e512 with expected crc 2024-03-29T05:01:05.361 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[101970]: failed to encode map e512 with expected crc 2024-03-29T05:01:05.361 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[106466]: failed to encode map e511 with expected crc 2024-03-29T05:01:05.361 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[106466]: failed to encode map e511 with expected crc 2024-03-29T05:01:05.361 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[106466]: failed to encode map e511 with expected crc 2024-03-29T05:01:05.361 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[106466]: pgmap v431: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 8.7 KiB/s wr, 158 op/s 2024-03-29T05:01:05.361 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[106466]: failed to encode map e512 with expected crc 2024-03-29T05:01:05.361 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[106466]: osdmap e512: 8 total, 8 up, 8 in 2024-03-29T05:01:05.361 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[106466]: failed to encode map e512 with expected crc 2024-03-29T05:01:05.361 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[106466]: failed to encode map e512 with expected crc 2024-03-29T05:01:05.361 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:04 smithi181 ceph-mon[106466]: failed to encode map e512 with expected crc 2024-03-29T05:01:05.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:04 smithi187 ceph-mon[93536]: failed to encode map e511 with expected crc 2024-03-29T05:01:05.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:04 smithi187 ceph-mon[93536]: failed to encode map e511 with expected crc 2024-03-29T05:01:05.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:04 smithi187 ceph-mon[93536]: failed to encode map e511 with expected crc 2024-03-29T05:01:05.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:04 smithi187 ceph-mon[93536]: pgmap v431: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 8.7 KiB/s wr, 158 op/s 2024-03-29T05:01:05.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:04 smithi187 ceph-mon[93536]: failed to encode map e512 with expected crc 2024-03-29T05:01:05.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:04 smithi187 ceph-mon[93536]: osdmap e512: 8 total, 8 up, 8 in 2024-03-29T05:01:05.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:04 smithi187 ceph-mon[93536]: failed to encode map e512 with expected crc 2024-03-29T05:01:05.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:04 smithi187 ceph-mon[93536]: failed to encode map e512 with expected crc 2024-03-29T05:01:05.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:04 smithi187 ceph-mon[93536]: failed to encode map e512 with expected crc 2024-03-29T05:01:06.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:05 smithi187 ceph-mon[93536]: failed to encode map e512 with expected crc 2024-03-29T05:01:06.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:05 smithi187 ceph-mon[93536]: failed to encode map e512 with expected crc 2024-03-29T05:01:06.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:05 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1883380774' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:06.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:05 smithi187 ceph-mon[93536]: failed to encode map e513 with expected crc 2024-03-29T05:01:06.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:05 smithi187 ceph-mon[93536]: osdmap e513: 8 total, 8 up, 8 in 2024-03-29T05:01:06.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:05 smithi187 ceph-mon[93536]: failed to encode map e513 with expected crc 2024-03-29T05:01:06.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:05 smithi187 ceph-mon[93536]: failed to encode map e513 with expected crc 2024-03-29T05:01:06.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:05 smithi181 ceph-mon[106466]: failed to encode map e512 with expected crc 2024-03-29T05:01:06.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:05 smithi181 ceph-mon[106466]: failed to encode map e512 with expected crc 2024-03-29T05:01:06.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:05 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1883380774' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:06.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:05 smithi181 ceph-mon[106466]: failed to encode map e513 with expected crc 2024-03-29T05:01:06.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:05 smithi181 ceph-mon[106466]: osdmap e513: 8 total, 8 up, 8 in 2024-03-29T05:01:06.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:05 smithi181 ceph-mon[106466]: failed to encode map e513 with expected crc 2024-03-29T05:01:06.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:05 smithi181 ceph-mon[106466]: failed to encode map e513 with expected crc 2024-03-29T05:01:06.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:05 smithi181 ceph-mon[101970]: failed to encode map e512 with expected crc 2024-03-29T05:01:06.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:05 smithi181 ceph-mon[101970]: failed to encode map e512 with expected crc 2024-03-29T05:01:06.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:05 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1883380774' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:06.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:05 smithi181 ceph-mon[101970]: failed to encode map e513 with expected crc 2024-03-29T05:01:06.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:05 smithi181 ceph-mon[101970]: osdmap e513: 8 total, 8 up, 8 in 2024-03-29T05:01:06.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:05 smithi181 ceph-mon[101970]: failed to encode map e513 with expected crc 2024-03-29T05:01:06.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:05 smithi181 ceph-mon[101970]: failed to encode map e513 with expected crc 2024-03-29T05:01:07.249 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:06 smithi181 systemd[1]: Stopping Ceph osd.2 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:01:07.249 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:06 smithi181 ceph-mon[101970]: failed to encode map e513 with expected crc 2024-03-29T05:01:07.249 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:06 smithi181 ceph-mon[101970]: failed to encode map e513 with expected crc 2024-03-29T05:01:07.249 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:06 smithi181 ceph-mon[101970]: pgmap v434: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:01:07.249 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:06 smithi181 ceph-mon[101970]: osdmap e514: 8 total, 8 up, 8 in 2024-03-29T05:01:07.249 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:06 smithi181 ceph-mon[101970]: failed to encode map e514 with expected crc 2024-03-29T05:01:07.250 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:06 smithi181 ceph-mon[101970]: failed to encode map e514 with expected crc 2024-03-29T05:01:07.250 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:06 smithi181 ceph-mon[106466]: failed to encode map e513 with expected crc 2024-03-29T05:01:07.250 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:06 smithi181 ceph-mon[106466]: failed to encode map e513 with expected crc 2024-03-29T05:01:07.250 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:06 smithi181 ceph-mon[106466]: pgmap v434: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:01:07.250 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:06 smithi181 ceph-mon[106466]: osdmap e514: 8 total, 8 up, 8 in 2024-03-29T05:01:07.250 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:06 smithi181 ceph-mon[106466]: failed to encode map e514 with expected crc 2024-03-29T05:01:07.250 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:06 smithi181 ceph-mon[106466]: failed to encode map e514 with expected crc 2024-03-29T05:01:07.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:06 smithi187 ceph-mon[93536]: failed to encode map e513 with expected crc 2024-03-29T05:01:07.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:06 smithi187 ceph-mon[93536]: failed to encode map e513 with expected crc 2024-03-29T05:01:07.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:06 smithi187 ceph-mon[93536]: pgmap v434: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:01:07.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:06 smithi187 ceph-mon[93536]: osdmap e514: 8 total, 8 up, 8 in 2024-03-29T05:01:07.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:06 smithi187 ceph-mon[93536]: failed to encode map e514 with expected crc 2024-03-29T05:01:07.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:06 smithi187 ceph-mon[93536]: failed to encode map e514 with expected crc 2024-03-29T05:01:07.704 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2[57869]: 2024-03-29T05:01:07.239+0000 7f682bdcc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T05:01:07.704 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2[57869]: 2024-03-29T05:01:07.239+0000 7f682bdcc700 -1 osd.2 514 *** Got signal Terminated *** 2024-03-29T05:01:07.704 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2[57869]: 2024-03-29T05:01:07.239+0000 7f682bdcc700 -1 osd.2 514 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T05:01:08.331 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:08 smithi181 podman[126283]: 2024-03-29 05:01:08.035845457 +0000 UTC m=+0.944916428 container died 9c61fb0d684c34e27192fc2e45d14a7dc525058005026f89686acebdbce0f686 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-03-29T05:01:08.331 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-mon[101970]: failed to encode map e514 with expected crc 2024-03-29T05:01:08.331 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-mon[101970]: failed to encode map e514 with expected crc 2024-03-29T05:01:08.331 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-mon[101970]: osd.2 marked itself down and dead 2024-03-29T05:01:08.331 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-mon[101970]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T05:01:08.331 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-mon[101970]: osdmap e515: 8 total, 7 up, 8 in 2024-03-29T05:01:08.331 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-mon[101970]: failed to encode map e515 with expected crc 2024-03-29T05:01:08.331 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-mon[101970]: failed to encode map e515 with expected crc 2024-03-29T05:01:08.332 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/417516562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:08.332 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-mon[106466]: failed to encode map e514 with expected crc 2024-03-29T05:01:08.332 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-mon[106466]: failed to encode map e514 with expected crc 2024-03-29T05:01:08.332 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-mon[106466]: osd.2 marked itself down and dead 2024-03-29T05:01:08.332 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-mon[106466]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T05:01:08.332 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-mon[106466]: osdmap e515: 8 total, 7 up, 8 in 2024-03-29T05:01:08.332 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-mon[106466]: failed to encode map e515 with expected crc 2024-03-29T05:01:08.333 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-mon[106466]: failed to encode map e515 with expected crc 2024-03-29T05:01:08.333 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:07 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/417516562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:08.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:07 smithi187 ceph-mon[93536]: failed to encode map e514 with expected crc 2024-03-29T05:01:08.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:07 smithi187 ceph-mon[93536]: failed to encode map e514 with expected crc 2024-03-29T05:01:08.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:07 smithi187 ceph-mon[93536]: osd.2 marked itself down and dead 2024-03-29T05:01:08.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:07 smithi187 ceph-mon[93536]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T05:01:08.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:07 smithi187 ceph-mon[93536]: osdmap e515: 8 total, 7 up, 8 in 2024-03-29T05:01:08.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:07 smithi187 ceph-mon[93536]: failed to encode map e515 with expected crc 2024-03-29T05:01:08.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:07 smithi187 ceph-mon[93536]: failed to encode map e515 with expected crc 2024-03-29T05:01:08.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:07 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/417516562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:08.704 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:08 smithi181 podman[126283]: 2024-03-29 05:01:08.438476603 +0000 UTC m=+1.347547574 container cleanup 9c61fb0d684c34e27192fc2e45d14a7dc525058005026f89686acebdbce0f686 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=quincy-4213e17, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_CLEAN=True) 2024-03-29T05:01:08.704 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:08 smithi181 bash[126283]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2 2024-03-29T05:01:09.007 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:01:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:01:08] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2024-03-29T05:01:09.008 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:08 smithi181 podman[126332]: 2024-03-29 05:01:08.72345741 +0000 UTC m=+0.686474730 container remove 9c61fb0d684c34e27192fc2e45d14a7dc525058005026f89686acebdbce0f686 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-29T05:01:09.302 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:08 smithi181 ceph-mon[106466]: failed to encode map e515 with expected crc 2024-03-29T05:01:09.302 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[106466]: failed to encode map e515 with expected crc 2024-03-29T05:01:09.302 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[106466]: failed to encode map e515 with expected crc 2024-03-29T05:01:09.302 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[106466]: pgmap v437: 105 pgs: 9 stale+active+clean, 96 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.5 KiB/s wr, 118 op/s 2024-03-29T05:01:09.303 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[106466]: osdmap e516: 8 total, 7 up, 8 in 2024-03-29T05:01:09.303 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[106466]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.303 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[106466]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.303 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[106466]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.303 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[106466]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.303 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[106466]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.304 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[106466]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.304 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:08 smithi181 ceph-mon[101970]: failed to encode map e515 with expected crc 2024-03-29T05:01:09.304 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[101970]: failed to encode map e515 with expected crc 2024-03-29T05:01:09.304 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[101970]: failed to encode map e515 with expected crc 2024-03-29T05:01:09.305 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[101970]: pgmap v437: 105 pgs: 9 stale+active+clean, 96 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.5 KiB/s wr, 118 op/s 2024-03-29T05:01:09.305 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[101970]: osdmap e516: 8 total, 7 up, 8 in 2024-03-29T05:01:09.305 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[101970]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.305 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[101970]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.305 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[101970]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.305 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[101970]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.306 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[101970]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.306 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:09 smithi181 ceph-mon[101970]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.306 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:09 smithi181 podman[126428]: 2024-03-29 05:01:08.957404962 +0000 UTC m=+0.018198174 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:01:09.306 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:09 smithi181 podman[126428]: 2024-03-29 05:01:09.057931813 +0000 UTC m=+0.118725026 container create 11d3bf9a232e5d3c841538459e3d390ec4a86ec0d0aa84c914b90979c95069fd (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=) 2024-03-29T05:01:09.307 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:09 smithi181 podman[126428]: 2024-03-29 05:01:09.293876692 +0000 UTC m=+0.354669908 container init 11d3bf9a232e5d3c841538459e3d390ec4a86ec0d0aa84c914b90979c95069fd (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-deactivate, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T05:01:09.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:08 smithi187 ceph-mon[93536]: failed to encode map e515 with expected crc 2024-03-29T05:01:09.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:08 smithi187 ceph-mon[93536]: failed to encode map e515 with expected crc 2024-03-29T05:01:09.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:08 smithi187 ceph-mon[93536]: failed to encode map e515 with expected crc 2024-03-29T05:01:09.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:09 smithi187 ceph-mon[93536]: pgmap v437: 105 pgs: 9 stale+active+clean, 96 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.5 KiB/s wr, 118 op/s 2024-03-29T05:01:09.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:09 smithi187 ceph-mon[93536]: osdmap e516: 8 total, 7 up, 8 in 2024-03-29T05:01:09.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:09 smithi187 ceph-mon[93536]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:09 smithi187 ceph-mon[93536]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:09 smithi187 ceph-mon[93536]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:09 smithi187 ceph-mon[93536]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:09 smithi187 ceph-mon[93536]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:09 smithi187 ceph-mon[93536]: failed to encode map e516 with expected crc 2024-03-29T05:01:09.606 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:09 smithi181 podman[126428]: 2024-03-29 05:01:09.299809177 +0000 UTC m=+0.360602390 container start 11d3bf9a232e5d3c841538459e3d390ec4a86ec0d0aa84c914b90979c95069fd (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=) 2024-03-29T05:01:09.606 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:09 smithi181 podman[126428]: 2024-03-29 05:01:09.410678243 +0000 UTC m=+0.471471465 container attach 11d3bf9a232e5d3c841538459e3d390ec4a86ec0d0aa84c914b90979c95069fd (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-03-29T05:01:09.914 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:09 smithi181 podman[126473]: 2024-03-29 05:01:09.626344012 +0000 UTC m=+0.018952610 container died 11d3bf9a232e5d3c841538459e3d390ec4a86ec0d0aa84c914b90979c95069fd (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-03-29T05:01:10.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:10 smithi181 ceph-mon[106466]: failed to encode map e516 with expected crc 2024-03-29T05:01:10.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:10 smithi181 ceph-mon[106466]: failed to encode map e516 with expected crc 2024-03-29T05:01:10.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:10 smithi181 ceph-mon[106466]: failed to encode map e516 with expected crc 2024-03-29T05:01:10.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:10 smithi181 ceph-mon[106466]: failed to encode map e516 with expected crc 2024-03-29T05:01:10.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:10 smithi181 ceph-mon[106466]: osdmap e517: 8 total, 7 up, 8 in 2024-03-29T05:01:10.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:10 smithi181 ceph-mon[106466]: failed to encode map e517 with expected crc 2024-03-29T05:01:10.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:10 smithi181 ceph-mon[101970]: failed to encode map e516 with expected crc 2024-03-29T05:01:10.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:10 smithi181 ceph-mon[101970]: failed to encode map e516 with expected crc 2024-03-29T05:01:10.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:10 smithi181 ceph-mon[101970]: failed to encode map e516 with expected crc 2024-03-29T05:01:10.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:10 smithi181 ceph-mon[101970]: failed to encode map e516 with expected crc 2024-03-29T05:01:10.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:10 smithi181 ceph-mon[101970]: osdmap e517: 8 total, 7 up, 8 in 2024-03-29T05:01:10.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:10 smithi181 ceph-mon[101970]: failed to encode map e517 with expected crc 2024-03-29T05:01:10.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:10 smithi187 ceph-mon[93536]: failed to encode map e516 with expected crc 2024-03-29T05:01:10.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:10 smithi187 ceph-mon[93536]: failed to encode map e516 with expected crc 2024-03-29T05:01:10.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:10 smithi187 ceph-mon[93536]: failed to encode map e516 with expected crc 2024-03-29T05:01:10.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:10 smithi187 ceph-mon[93536]: failed to encode map e516 with expected crc 2024-03-29T05:01:10.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:10 smithi187 ceph-mon[93536]: osdmap e517: 8 total, 7 up, 8 in 2024-03-29T05:01:10.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:10 smithi187 ceph-mon[93536]: failed to encode map e517 with expected crc 2024-03-29T05:01:10.520 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:10 smithi181 podman[126473]: 2024-03-29 05:01:10.210178297 +0000 UTC m=+0.602786898 container remove 11d3bf9a232e5d3c841538459e3d390ec4a86ec0d0aa84c914b90979c95069fd (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-deactivate, GIT_CLEAN=True, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-03-29T05:01:10.520 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:10 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.2.service: Deactivated successfully. 2024-03-29T05:01:10.520 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:10 smithi181 systemd[1]: Stopped Ceph osd.2 for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T05:01:10.520 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:10 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.2.service: Consumed 23.007s CPU time. 2024-03-29T05:01:10.520 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:10 smithi181 systemd[1]: Starting Ceph osd.2 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:01:10.844 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:10 smithi181 podman[126607]: 2024-03-29 05:01:10.585257154 +0000 UTC m=+0.021448957 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:01:10.844 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:10 smithi181 podman[126607]: 2024-03-29 05:01:10.70795454 +0000 UTC m=+0.144146323 container create 7090fc5746e75a78d98f40b0b036a1c7ab93e3b0a96f4309559e894c50ce2496 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-activate, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-03-29T05:01:11.204 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:10 smithi181 podman[126607]: 2024-03-29 05:01:10.93565504 +0000 UTC m=+0.371846826 container init 7090fc5746e75a78d98f40b0b036a1c7ab93e3b0a96f4309559e894c50ce2496 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-activate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux ) 2024-03-29T05:01:11.205 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:10 smithi181 podman[126607]: 2024-03-29 05:01:10.941199818 +0000 UTC m=+0.377391602 container start 7090fc5746e75a78d98f40b0b036a1c7ab93e3b0a96f4309559e894c50ce2496 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-03-29T05:01:11.205 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 podman[126607]: 2024-03-29 05:01:11.043989058 +0000 UTC m=+0.480180843 container attach 7090fc5746e75a78d98f40b0b036a1c7ab93e3b0a96f4309559e894c50ce2496 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326) 2024-03-29T05:01:11.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-mon[101970]: failed to encode map e517 with expected crc 2024-03-29T05:01:11.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-mon[101970]: failed to encode map e517 with expected crc 2024-03-29T05:01:11.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-mon[101970]: failed to encode map e517 with expected crc 2024-03-29T05:01:11.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2292938417' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:11.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-mon[101970]: pgmap v440: 105 pgs: 9 stale+active+clean, 96 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.5 KiB/s wr, 118 op/s 2024-03-29T05:01:11.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-mon[101970]: osdmap e518: 8 total, 7 up, 8 in 2024-03-29T05:01:11.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-mon[101970]: failed to encode map e518 with expected crc 2024-03-29T05:01:11.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-mon[101970]: failed to encode map e518 with expected crc 2024-03-29T05:01:11.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-mon[106466]: failed to encode map e517 with expected crc 2024-03-29T05:01:11.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-mon[106466]: failed to encode map e517 with expected crc 2024-03-29T05:01:11.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-mon[106466]: failed to encode map e517 with expected crc 2024-03-29T05:01:11.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2292938417' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:11.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-mon[106466]: pgmap v440: 105 pgs: 9 stale+active+clean, 96 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.5 KiB/s wr, 118 op/s 2024-03-29T05:01:11.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-mon[106466]: osdmap e518: 8 total, 7 up, 8 in 2024-03-29T05:01:11.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-mon[106466]: failed to encode map e518 with expected crc 2024-03-29T05:01:11.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-mon[106466]: failed to encode map e518 with expected crc 2024-03-29T05:01:11.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:11 smithi187 ceph-mon[93536]: failed to encode map e517 with expected crc 2024-03-29T05:01:11.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:11 smithi187 ceph-mon[93536]: failed to encode map e517 with expected crc 2024-03-29T05:01:11.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:11 smithi187 ceph-mon[93536]: failed to encode map e517 with expected crc 2024-03-29T05:01:11.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:11 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2292938417' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:11.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:11 smithi187 ceph-mon[93536]: pgmap v440: 105 pgs: 9 stale+active+clean, 96 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 7.5 KiB/s wr, 118 op/s 2024-03-29T05:01:11.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:11 smithi187 ceph-mon[93536]: osdmap e518: 8 total, 7 up, 8 in 2024-03-29T05:01:11.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:11 smithi187 ceph-mon[93536]: failed to encode map e518 with expected crc 2024-03-29T05:01:11.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:11 smithi187 ceph-mon[93536]: failed to encode map e518 with expected crc 2024-03-29T05:01:11.705 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-activate[126630]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-29T05:01:11.705 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-activate[126630]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-29T05:01:11.705 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 bash[126607]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-29T05:01:11.705 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 bash[126607]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-29T05:01:11.705 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 bash[126607]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-29T05:01:11.705 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 bash[126607]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-29T05:01:11.705 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 bash[126607]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-03-29T05:01:11.705 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 bash[126607]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-29T05:01:11.705 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 bash[126607]: --> ceph-volume raw activate successful for osd ID: 2 2024-03-29T05:01:11.705 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-activate[126630]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-29T05:01:11.705 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-activate[126630]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-29T05:01:11.706 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-activate[126630]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-03-29T05:01:11.706 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-activate[126630]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-29T05:01:11.706 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-activate[126630]: --> ceph-volume raw activate successful for osd ID: 2 2024-03-29T05:01:11.706 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 podman[126607]: 2024-03-29 05:01:11.422174688 +0000 UTC m=+0.858366472 container died 7090fc5746e75a78d98f40b0b036a1c7ab93e3b0a96f4309559e894c50ce2496 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T05:01:12.331 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:11 smithi181 podman[126784]: 2024-03-29 05:01:11.973164623 +0000 UTC m=+0.545928016 container remove 7090fc5746e75a78d98f40b0b036a1c7ab93e3b0a96f4309559e894c50ce2496 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-activate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, io.buildah.version=1.33.5, ceph=True) 2024-03-29T05:01:12.331 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:12 smithi181 podman[126854]: 2024-03-29 05:01:12.088421712 +0000 UTC m=+0.018237465 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:01:12.331 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:12 smithi181 podman[126854]: 2024-03-29 05:01:12.207678803 +0000 UTC m=+0.137494557 container create 12c7ca8de2265669d61f2a3cbf5283ed2f12a3e85fa67cea9fae40f5e1b20b5b (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-03-29T05:01:12.331 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: failed to encode map e518 with expected crc 2024-03-29T05:01:12.331 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: failed to encode map e518 with expected crc 2024-03-29T05:01:12.331 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.2", "id": [1, 2]}]: dispatch 2024-03-29T05:01:12.331 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.6", "id": [6, 2]}]: dispatch 2024-03-29T05:01:12.332 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.9", "id": [1, 7]}]: dispatch 2024-03-29T05:01:12.332 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.f", "id": [0, 2]}]: dispatch 2024-03-29T05:01:12.332 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.10", "id": [0, 3]}]: dispatch 2024-03-29T05:01:12.332 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.12", "id": [0, 4]}]: dispatch 2024-03-29T05:01:12.332 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.16", "id": [1, 4]}]: dispatch 2024-03-29T05:01:12.332 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.19", "id": [1, 2]}]: dispatch 2024-03-29T05:01:12.332 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.1c", "id": [1, 4]}]: dispatch 2024-03-29T05:01:12.332 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.1f", "id": [5, 3]}]: dispatch 2024-03-29T05:01:12.332 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:12.332 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:01:12.332 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.2", "id": [1, 2]}]': finished 2024-03-29T05:01:12.332 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.6", "id": [6, 2]}]': finished 2024-03-29T05:01:12.332 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.9", "id": [1, 7]}]': finished 2024-03-29T05:01:12.332 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.f", "id": [0, 2]}]': finished 2024-03-29T05:01:12.333 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.10", "id": [0, 3]}]': finished 2024-03-29T05:01:12.333 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.12", "id": [0, 4]}]': finished 2024-03-29T05:01:12.333 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.16", "id": [1, 4]}]': finished 2024-03-29T05:01:12.333 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.19", "id": [1, 2]}]': finished 2024-03-29T05:01:12.333 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: failed to encode map e518 with expected crc 2024-03-29T05:01:12.333 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: failed to encode map e518 with expected crc 2024-03-29T05:01:12.333 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.2", "id": [1, 2]}]: dispatch 2024-03-29T05:01:12.333 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.6", "id": [6, 2]}]: dispatch 2024-03-29T05:01:12.333 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.9", "id": [1, 7]}]: dispatch 2024-03-29T05:01:12.333 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.f", "id": [0, 2]}]: dispatch 2024-03-29T05:01:12.333 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.10", "id": [0, 3]}]: dispatch 2024-03-29T05:01:12.333 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.12", "id": [0, 4]}]: dispatch 2024-03-29T05:01:12.334 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.16", "id": [1, 4]}]: dispatch 2024-03-29T05:01:12.334 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.19", "id": [1, 2]}]: dispatch 2024-03-29T05:01:12.334 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.1c", "id": [1, 4]}]: dispatch 2024-03-29T05:01:12.334 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.1f", "id": [5, 3]}]: dispatch 2024-03-29T05:01:12.334 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:12.334 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:01:12.334 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.2", "id": [1, 2]}]': finished 2024-03-29T05:01:12.334 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.6", "id": [6, 2]}]': finished 2024-03-29T05:01:12.334 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.9", "id": [1, 7]}]': finished 2024-03-29T05:01:12.334 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.f", "id": [0, 2]}]': finished 2024-03-29T05:01:12.334 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.10", "id": [0, 3]}]': finished 2024-03-29T05:01:12.334 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.12", "id": [0, 4]}]': finished 2024-03-29T05:01:12.334 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.16", "id": [1, 4]}]': finished 2024-03-29T05:01:12.335 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.19", "id": [1, 2]}]': finished 2024-03-29T05:01:12.335 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.1c", "id": [1, 4]}]': finished 2024-03-29T05:01:12.335 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.1f", "id": [5, 3]}]': finished 2024-03-29T05:01:12.335 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: osdmap e519: 8 total, 7 up, 8 in 2024-03-29T05:01:12.335 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[101970]: failed to encode map e519 with expected crc 2024-03-29T05:01:12.335 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.1c", "id": [1, 4]}]': finished 2024-03-29T05:01:12.335 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.1f", "id": [5, 3]}]': finished 2024-03-29T05:01:12.335 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: osdmap e519: 8 total, 7 up, 8 in 2024-03-29T05:01:12.336 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-mon[106466]: failed to encode map e519 with expected crc 2024-03-29T05:01:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: failed to encode map e518 with expected crc 2024-03-29T05:01:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: failed to encode map e518 with expected crc 2024-03-29T05:01:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.2", "id": [1, 2]}]: dispatch 2024-03-29T05:01:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.6", "id": [6, 2]}]: dispatch 2024-03-29T05:01:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.9", "id": [1, 7]}]: dispatch 2024-03-29T05:01:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.f", "id": [0, 2]}]: dispatch 2024-03-29T05:01:12.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.10", "id": [0, 3]}]: dispatch 2024-03-29T05:01:12.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.12", "id": [0, 4]}]: dispatch 2024-03-29T05:01:12.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.16", "id": [1, 4]}]: dispatch 2024-03-29T05:01:12.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.19", "id": [1, 2]}]: dispatch 2024-03-29T05:01:12.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.1c", "id": [1, 4]}]: dispatch 2024-03-29T05:01:12.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.1f", "id": [5, 3]}]: dispatch 2024-03-29T05:01:12.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:12.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:01:12.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.2", "id": [1, 2]}]': finished 2024-03-29T05:01:12.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.6", "id": [6, 2]}]': finished 2024-03-29T05:01:12.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.9", "id": [1, 7]}]': finished 2024-03-29T05:01:12.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.f", "id": [0, 2]}]': finished 2024-03-29T05:01:12.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.10", "id": [0, 3]}]': finished 2024-03-29T05:01:12.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.12", "id": [0, 4]}]': finished 2024-03-29T05:01:12.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.16", "id": [1, 4]}]': finished 2024-03-29T05:01:12.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.19", "id": [1, 2]}]': finished 2024-03-29T05:01:12.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.1c", "id": [1, 4]}]': finished 2024-03-29T05:01:12.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "16.1f", "id": [5, 3]}]': finished 2024-03-29T05:01:12.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: osdmap e519: 8 total, 7 up, 8 in 2024-03-29T05:01:12.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:12 smithi187 ceph-mon[93536]: failed to encode map e519 with expected crc 2024-03-29T05:01:12.954 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:12 smithi181 podman[126854]: 2024-03-29 05:01:12.452291832 +0000 UTC m=+0.382107592 container init 12c7ca8de2265669d61f2a3cbf5283ed2f12a3e85fa67cea9fae40f5e1b20b5b (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-03-29T05:01:12.955 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:12 smithi181 podman[126854]: 2024-03-29 05:01:12.457366237 +0000 UTC m=+0.387181991 container start 12c7ca8de2265669d61f2a3cbf5283ed2f12a3e85fa67cea9fae40f5e1b20b5b (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-03-29T05:01:12.955 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:12 smithi181 ceph-osd[126918]: -- 172.21.15.181:0/1814182305 <== mon.2 v2:172.21.15.181:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5639257e7680 con 0x5639257b1000 2024-03-29T05:01:12.955 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:12 smithi181 bash[126854]: 12c7ca8de2265669d61f2a3cbf5283ed2f12a3e85fa67cea9fae40f5e1b20b5b 2024-03-29T05:01:12.955 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:12 smithi181 systemd[1]: Started Ceph osd.2 for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T05:01:13.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:13 smithi187 ceph-mon[93536]: failed to encode map e519 with expected crc 2024-03-29T05:01:13.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:13 smithi187 ceph-mon[93536]: failed to encode map e519 with expected crc 2024-03-29T05:01:13.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:13 smithi187 ceph-mon[93536]: pgmap v443: 105 pgs: 6 peering, 13 active+undersized+degraded, 17 active+undersized, 69 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 5.5 KiB/s wr, 102 op/s; 38/330 objects degraded (11.515%) 2024-03-29T05:01:13.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:13 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2562504736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:13.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:13 smithi187 ceph-mon[93536]: failed to encode map e520 with expected crc 2024-03-29T05:01:13.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:13 smithi187 ceph-mon[93536]: osdmap e520: 8 total, 7 up, 8 in 2024-03-29T05:01:13.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:13 smithi187 ceph-mon[93536]: failed to encode map e520 with expected crc 2024-03-29T05:01:13.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:13 smithi187 ceph-mon[93536]: failed to encode map e520 with expected crc 2024-03-29T05:01:13.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:13.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:13.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:01:13.454 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2[126892]: 2024-03-29T05:01:13.000+0000 7fe3af8dd7c0 -1 Falling back to public interface 2024-03-29T05:01:13.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[106466]: failed to encode map e519 with expected crc 2024-03-29T05:01:13.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[106466]: failed to encode map e519 with expected crc 2024-03-29T05:01:13.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[106466]: pgmap v443: 105 pgs: 6 peering, 13 active+undersized+degraded, 17 active+undersized, 69 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 5.5 KiB/s wr, 102 op/s; 38/330 objects degraded (11.515%) 2024-03-29T05:01:13.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2562504736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:13.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[106466]: failed to encode map e520 with expected crc 2024-03-29T05:01:13.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[106466]: osdmap e520: 8 total, 7 up, 8 in 2024-03-29T05:01:13.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[106466]: failed to encode map e520 with expected crc 2024-03-29T05:01:13.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[106466]: failed to encode map e520 with expected crc 2024-03-29T05:01:13.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:13.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:13.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:01:13.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[101970]: failed to encode map e519 with expected crc 2024-03-29T05:01:13.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[101970]: failed to encode map e519 with expected crc 2024-03-29T05:01:13.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[101970]: pgmap v443: 105 pgs: 6 peering, 13 active+undersized+degraded, 17 active+undersized, 69 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 5.5 KiB/s wr, 102 op/s; 38/330 objects degraded (11.515%) 2024-03-29T05:01:13.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2562504736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:13.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[101970]: failed to encode map e520 with expected crc 2024-03-29T05:01:13.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[101970]: osdmap e520: 8 total, 7 up, 8 in 2024-03-29T05:01:13.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[101970]: failed to encode map e520 with expected crc 2024-03-29T05:01:13.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[101970]: failed to encode map e520 with expected crc 2024-03-29T05:01:13.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:13.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:13.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:01:14.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:14 smithi187 ceph-mon[93536]: failed to encode map e520 with expected crc 2024-03-29T05:01:14.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:14 smithi187 ceph-mon[93536]: failed to encode map e520 with expected crc 2024-03-29T05:01:14.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:14 smithi187 ceph-mon[93536]: failed to encode map e520 with expected crc 2024-03-29T05:01:14.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:14 smithi187 ceph-mon[93536]: failed to encode map e520 with expected crc 2024-03-29T05:01:14.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:14 smithi187 ceph-mon[93536]: Health check failed: Degraded data redundancy: 38/330 objects degraded (11.515%), 13 pgs degraded (PG_DEGRADED) 2024-03-29T05:01:14.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:14 smithi187 ceph-mon[93536]: failed to encode map e521 with expected crc 2024-03-29T05:01:14.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:14 smithi187 ceph-mon[93536]: osdmap e521: 8 total, 7 up, 8 in 2024-03-29T05:01:14.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:14 smithi187 ceph-mon[93536]: failed to encode map e521 with expected crc 2024-03-29T05:01:14.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:14 smithi187 ceph-mon[93536]: failed to encode map e521 with expected crc 2024-03-29T05:01:14.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:14 smithi187 ceph-mon[93536]: failed to encode map e521 with expected crc 2024-03-29T05:01:14.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[101970]: failed to encode map e520 with expected crc 2024-03-29T05:01:14.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[101970]: failed to encode map e520 with expected crc 2024-03-29T05:01:14.425 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[101970]: failed to encode map e520 with expected crc 2024-03-29T05:01:14.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[101970]: failed to encode map e520 with expected crc 2024-03-29T05:01:14.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[101970]: Health check failed: Degraded data redundancy: 38/330 objects degraded (11.515%), 13 pgs degraded (PG_DEGRADED) 2024-03-29T05:01:14.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[101970]: failed to encode map e521 with expected crc 2024-03-29T05:01:14.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[101970]: osdmap e521: 8 total, 7 up, 8 in 2024-03-29T05:01:14.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[101970]: failed to encode map e521 with expected crc 2024-03-29T05:01:14.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[101970]: failed to encode map e521 with expected crc 2024-03-29T05:01:14.426 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[101970]: failed to encode map e521 with expected crc 2024-03-29T05:01:14.426 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[106466]: failed to encode map e520 with expected crc 2024-03-29T05:01:14.426 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[106466]: failed to encode map e520 with expected crc 2024-03-29T05:01:14.426 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[106466]: failed to encode map e520 with expected crc 2024-03-29T05:01:14.426 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[106466]: failed to encode map e520 with expected crc 2024-03-29T05:01:14.426 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[106466]: Health check failed: Degraded data redundancy: 38/330 objects degraded (11.515%), 13 pgs degraded (PG_DEGRADED) 2024-03-29T05:01:14.427 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[106466]: failed to encode map e521 with expected crc 2024-03-29T05:01:14.427 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[106466]: osdmap e521: 8 total, 7 up, 8 in 2024-03-29T05:01:14.427 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[106466]: failed to encode map e521 with expected crc 2024-03-29T05:01:14.427 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[106466]: failed to encode map e521 with expected crc 2024-03-29T05:01:14.427 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:14 smithi181 ceph-mon[106466]: failed to encode map e521 with expected crc 2024-03-29T05:01:15.322 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-mon[101970]: failed to encode map e521 with expected crc 2024-03-29T05:01:15.322 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-mon[101970]: failed to encode map e521 with expected crc 2024-03-29T05:01:15.322 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-mon[101970]: failed to encode map e521 with expected crc 2024-03-29T05:01:15.322 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-mon[101970]: pgmap v446: 105 pgs: 6 peering, 13 active+undersized+degraded, 17 active+undersized, 69 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 5.5 KiB/s wr, 102 op/s; 38/330 objects degraded (11.515%) 2024-03-29T05:01:15.323 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-mon[101970]: osdmap e522: 8 total, 7 up, 8 in 2024-03-29T05:01:15.323 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-mon[101970]: failed to encode map e522 with expected crc 2024-03-29T05:01:15.323 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-mon[101970]: failed to encode map e522 with expected crc 2024-03-29T05:01:15.323 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2382665196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:15.323 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-mon[106466]: failed to encode map e521 with expected crc 2024-03-29T05:01:15.323 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-mon[106466]: failed to encode map e521 with expected crc 2024-03-29T05:01:15.323 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-mon[106466]: failed to encode map e521 with expected crc 2024-03-29T05:01:15.323 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-mon[106466]: pgmap v446: 105 pgs: 6 peering, 13 active+undersized+degraded, 17 active+undersized, 69 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 5.5 KiB/s wr, 102 op/s; 38/330 objects degraded (11.515%) 2024-03-29T05:01:15.323 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-mon[106466]: osdmap e522: 8 total, 7 up, 8 in 2024-03-29T05:01:15.323 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-mon[106466]: failed to encode map e522 with expected crc 2024-03-29T05:01:15.323 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-mon[106466]: failed to encode map e522 with expected crc 2024-03-29T05:01:15.323 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2382665196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:15.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:15 smithi187 ceph-mon[93536]: failed to encode map e521 with expected crc 2024-03-29T05:01:15.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:15 smithi187 ceph-mon[93536]: failed to encode map e521 with expected crc 2024-03-29T05:01:15.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:15 smithi187 ceph-mon[93536]: failed to encode map e521 with expected crc 2024-03-29T05:01:15.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:15 smithi187 ceph-mon[93536]: pgmap v446: 105 pgs: 6 peering, 13 active+undersized+degraded, 17 active+undersized, 69 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 5.5 KiB/s wr, 102 op/s; 38/330 objects degraded (11.515%) 2024-03-29T05:01:15.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:15 smithi187 ceph-mon[93536]: osdmap e522: 8 total, 7 up, 8 in 2024-03-29T05:01:15.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:15 smithi187 ceph-mon[93536]: failed to encode map e522 with expected crc 2024-03-29T05:01:15.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:15 smithi187 ceph-mon[93536]: failed to encode map e522 with expected crc 2024-03-29T05:01:15.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:15 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2382665196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:16.113 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:15 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2[126892]: 2024-03-29T05:01:15.836+0000 7fe3af8dd7c0 -1 osd.2 514 log_to_monitors true 2024-03-29T05:01:16.113 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:16 smithi181 ceph-mon[106466]: failed to encode map e522 with expected crc 2024-03-29T05:01:16.113 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:16 smithi181 ceph-mon[106466]: failed to encode map e522 with expected crc 2024-03-29T05:01:16.113 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:16 smithi181 ceph-mon[106466]: failed to encode map e522 with expected crc 2024-03-29T05:01:16.113 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:16 smithi181 ceph-mon[106466]: osdmap e523: 8 total, 7 up, 8 in 2024-03-29T05:01:16.113 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:16 smithi181 ceph-mon[106466]: failed to encode map e523 with expected crc 2024-03-29T05:01:16.113 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:16 smithi181 ceph-mon[106466]: from='osd.2 [v2:172.21.15.181:6818/1623440102,v1:172.21.15.181:6819/1623440102]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T05:01:16.113 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:16 smithi181 ceph-mon[106466]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T05:01:16.114 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:16 smithi181 ceph-mon[101970]: failed to encode map e522 with expected crc 2024-03-29T05:01:16.114 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:16 smithi181 ceph-mon[101970]: failed to encode map e522 with expected crc 2024-03-29T05:01:16.114 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:16 smithi181 ceph-mon[101970]: failed to encode map e522 with expected crc 2024-03-29T05:01:16.114 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:16 smithi181 ceph-mon[101970]: osdmap e523: 8 total, 7 up, 8 in 2024-03-29T05:01:16.114 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:16 smithi181 ceph-mon[101970]: failed to encode map e523 with expected crc 2024-03-29T05:01:16.114 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:16 smithi181 ceph-mon[101970]: from='osd.2 [v2:172.21.15.181:6818/1623440102,v1:172.21.15.181:6819/1623440102]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T05:01:16.114 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:16 smithi181 ceph-mon[101970]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T05:01:16.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:16 smithi187 ceph-mon[93536]: failed to encode map e522 with expected crc 2024-03-29T05:01:16.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:16 smithi187 ceph-mon[93536]: failed to encode map e522 with expected crc 2024-03-29T05:01:16.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:16 smithi187 ceph-mon[93536]: failed to encode map e522 with expected crc 2024-03-29T05:01:16.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:16 smithi187 ceph-mon[93536]: osdmap e523: 8 total, 7 up, 8 in 2024-03-29T05:01:16.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:16 smithi187 ceph-mon[93536]: failed to encode map e523 with expected crc 2024-03-29T05:01:16.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:16 smithi187 ceph-mon[93536]: from='osd.2 [v2:172.21.15.181:6818/1623440102,v1:172.21.15.181:6819/1623440102]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T05:01:16.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:16 smithi187 ceph-mon[93536]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T05:01:16.876 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestMirroring.AioSnapshotCreate (34974 ms) 2024-03-29T05:01:17.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:17 smithi187 ceph-mon[93536]: failed to encode map e523 with expected crc 2024-03-29T05:01:17.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:17 smithi187 ceph-mon[93536]: failed to encode map e523 with expected crc 2024-03-29T05:01:17.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:17 smithi187 ceph-mon[93536]: failed to encode map e523 with expected crc 2024-03-29T05:01:17.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:17 smithi187 ceph-mon[93536]: failed to encode map e523 with expected crc 2024-03-29T05:01:17.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:17 smithi187 ceph-mon[93536]: failed to encode map e523 with expected crc 2024-03-29T05:01:17.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:17 smithi187 ceph-mon[93536]: pgmap v449: 105 pgs: 6 peering, 13 active+undersized+degraded, 17 active+undersized, 69 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 38/330 objects degraded (11.515%) 2024-03-29T05:01:17.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:17 smithi187 ceph-mon[93536]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T05:01:17.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:17 smithi187 ceph-mon[93536]: osdmap e524: 8 total, 7 up, 8 in 2024-03-29T05:01:17.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:17 smithi187 ceph-mon[93536]: from='osd.2 [v2:172.21.15.181:6818/1623440102,v1:172.21.15.181:6819/1623440102]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T05:01:17.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:17 smithi187 ceph-mon[93536]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T05:01:17.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:17 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:17.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:17 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:17.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:17 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/51052672-0aac-49e5-a2ba-8f1bd2d4ef50"}]: dispatch 2024-03-29T05:01:17.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:17 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/51052672-0aac-49e5-a2ba-8f1bd2d4ef50"}]': finished 2024-03-29T05:01:17.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:17 smithi187 ceph-mon[93536]: from='osd.2 ' entity='osd.2' 2024-03-29T05:01:17.454 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:01:16 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2[126892]: 2024-03-29T05:01:16.965+0000 7fe39cd2d700 -1 osd.2 514 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T05:01:17.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[106466]: failed to encode map e523 with expected crc 2024-03-29T05:01:17.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[106466]: failed to encode map e523 with expected crc 2024-03-29T05:01:17.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[106466]: failed to encode map e523 with expected crc 2024-03-29T05:01:17.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[106466]: failed to encode map e523 with expected crc 2024-03-29T05:01:17.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[106466]: failed to encode map e523 with expected crc 2024-03-29T05:01:17.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[106466]: pgmap v449: 105 pgs: 6 peering, 13 active+undersized+degraded, 17 active+undersized, 69 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 38/330 objects degraded (11.515%) 2024-03-29T05:01:17.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[106466]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T05:01:17.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[106466]: osdmap e524: 8 total, 7 up, 8 in 2024-03-29T05:01:17.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[106466]: from='osd.2 [v2:172.21.15.181:6818/1623440102,v1:172.21.15.181:6819/1623440102]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T05:01:17.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[106466]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T05:01:17.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:17.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:17.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/51052672-0aac-49e5-a2ba-8f1bd2d4ef50"}]: dispatch 2024-03-29T05:01:17.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/51052672-0aac-49e5-a2ba-8f1bd2d4ef50"}]': finished 2024-03-29T05:01:17.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[106466]: from='osd.2 ' entity='osd.2' 2024-03-29T05:01:17.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[101970]: failed to encode map e523 with expected crc 2024-03-29T05:01:17.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[101970]: failed to encode map e523 with expected crc 2024-03-29T05:01:17.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[101970]: failed to encode map e523 with expected crc 2024-03-29T05:01:17.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[101970]: failed to encode map e523 with expected crc 2024-03-29T05:01:17.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[101970]: failed to encode map e523 with expected crc 2024-03-29T05:01:17.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[101970]: pgmap v449: 105 pgs: 6 peering, 13 active+undersized+degraded, 17 active+undersized, 69 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 38/330 objects degraded (11.515%) 2024-03-29T05:01:17.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[101970]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T05:01:17.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[101970]: osdmap e524: 8 total, 7 up, 8 in 2024-03-29T05:01:17.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[101970]: from='osd.2 [v2:172.21.15.181:6818/1623440102,v1:172.21.15.181:6819/1623440102]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T05:01:17.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[101970]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T05:01:17.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:17.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:17.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/51052672-0aac-49e5-a2ba-8f1bd2d4ef50"}]: dispatch 2024-03-29T05:01:17.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/670815626' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/51052672-0aac-49e5-a2ba-8f1bd2d4ef50"}]': finished 2024-03-29T05:01:17.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:17 smithi181 ceph-mon[101970]: from='osd.2 ' entity='osd.2' 2024-03-29T05:01:17.512 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 51 tests from TestMirroring (92092 ms total) 2024-03-29T05:01:17.512 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:01:17.512 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 34 tests from TestInternal 2024-03-29T05:01:18.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:18 smithi187 ceph-mon[93536]: failed to encode map e524 with expected crc 2024-03-29T05:01:18.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:18 smithi187 ceph-mon[93536]: failed to encode map e524 with expected crc 2024-03-29T05:01:18.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:18 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1169852036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:18.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:18 smithi187 ceph-mon[93536]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T05:01:18.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:18 smithi187 ceph-mon[93536]: failed to encode map e525 with expected crc 2024-03-29T05:01:18.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:18 smithi187 ceph-mon[93536]: osd.2 [v2:172.21.15.181:6818/1623440102,v1:172.21.15.181:6819/1623440102] boot 2024-03-29T05:01:18.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:18 smithi187 ceph-mon[93536]: osdmap e525: 8 total, 8 up, 8 in 2024-03-29T05:01:18.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:18 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T05:01:18.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:18 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:18.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:18 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:18.407 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[101970]: failed to encode map e524 with expected crc 2024-03-29T05:01:18.407 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[101970]: failed to encode map e524 with expected crc 2024-03-29T05:01:18.407 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1169852036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:18.407 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[101970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T05:01:18.408 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[101970]: failed to encode map e525 with expected crc 2024-03-29T05:01:18.408 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[101970]: osd.2 [v2:172.21.15.181:6818/1623440102,v1:172.21.15.181:6819/1623440102] boot 2024-03-29T05:01:18.408 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[101970]: osdmap e525: 8 total, 8 up, 8 in 2024-03-29T05:01:18.408 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T05:01:18.408 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:18.408 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:18.408 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[106466]: failed to encode map e524 with expected crc 2024-03-29T05:01:18.408 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[106466]: failed to encode map e524 with expected crc 2024-03-29T05:01:18.408 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1169852036' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:18.408 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[106466]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T05:01:18.408 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[106466]: failed to encode map e525 with expected crc 2024-03-29T05:01:18.409 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[106466]: osd.2 [v2:172.21.15.181:6818/1623440102,v1:172.21.15.181:6819/1623440102] boot 2024-03-29T05:01:18.409 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[106466]: osdmap e525: 8 total, 8 up, 8 in 2024-03-29T05:01:18.409 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T05:01:18.409 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:18.409 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:19.454 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:01:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:01:18] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.43.0" 2024-03-29T05:01:19.513 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.OpenByID 2024-03-29T05:01:19.799 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[106466]: pgmap v452: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T05:01:19.799 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[106466]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.800 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[106466]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.800 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[106466]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.800 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[106466]: osdmap e526: 8 total, 8 up, 8 in 2024-03-29T05:01:19.800 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[106466]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.800 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[106466]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.800 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[106466]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.800 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[106466]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.800 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3239665902' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:01:19.800 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[106466]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T05:01:19.800 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/967301080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:19.800 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[101970]: pgmap v452: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T05:01:19.801 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[101970]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.801 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[101970]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.801 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[101970]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.801 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[101970]: osdmap e526: 8 total, 8 up, 8 in 2024-03-29T05:01:19.801 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[101970]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.801 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[101970]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.801 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[101970]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.801 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[101970]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.801 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3239665902' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:01:19.801 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[101970]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T05:01:19.801 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:19 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/967301080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:19.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:19 smithi187 ceph-mon[93536]: pgmap v452: 73 pgs: 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T05:01:19.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:19 smithi187 ceph-mon[93536]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:19 smithi187 ceph-mon[93536]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:19 smithi187 ceph-mon[93536]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:19 smithi187 ceph-mon[93536]: osdmap e526: 8 total, 8 up, 8 in 2024-03-29T05:01:19.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:19 smithi187 ceph-mon[93536]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:19 smithi187 ceph-mon[93536]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:19 smithi187 ceph-mon[93536]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:19 smithi187 ceph-mon[93536]: failed to encode map e526 with expected crc 2024-03-29T05:01:19.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:19 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3239665902' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:01:19.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:19 smithi187 ceph-mon[93536]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T05:01:19.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:19 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/967301080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:20.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:20 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3239665902' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:01:20.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:20 smithi187 ceph-mon[93536]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:20 smithi187 ceph-mon[93536]: osdmap e527: 8 total, 8 up, 8 in 2024-03-29T05:01:20.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:20 smithi187 ceph-mon[93536]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:20 smithi187 ceph-mon[93536]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:20 smithi187 ceph-mon[93536]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:20 smithi187 ceph-mon[93536]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:20 smithi187 ceph-mon[93536]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:20 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3239665902' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:01:20.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:20 smithi181 ceph-mon[101970]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:20 smithi181 ceph-mon[101970]: osdmap e527: 8 total, 8 up, 8 in 2024-03-29T05:01:20.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:20 smithi181 ceph-mon[101970]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:20 smithi181 ceph-mon[101970]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:20 smithi181 ceph-mon[101970]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:20 smithi181 ceph-mon[101970]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:20 smithi181 ceph-mon[101970]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:20 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3239665902' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:01:20.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:20 smithi181 ceph-mon[106466]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:20 smithi181 ceph-mon[106466]: osdmap e527: 8 total, 8 up, 8 in 2024-03-29T05:01:20.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:20 smithi181 ceph-mon[106466]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:20 smithi181 ceph-mon[106466]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:20 smithi181 ceph-mon[106466]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:20 smithi181 ceph-mon[106466]: failed to encode map e527 with expected crc 2024-03-29T05:01:20.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:20 smithi181 ceph-mon[106466]: failed to encode map e527 with expected crc 2024-03-29T05:01:21.558 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.OpenByID (2044 ms) 2024-03-29T05:01:21.558 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.OpenSnapDNE 2024-03-29T05:01:21.583 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.OpenSnapDNE (26 ms) 2024-03-29T05:01:21.583 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.IsExclusiveLockOwner 2024-03-29T05:01:21.612 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.IsExclusiveLockOwner (29 ms) 2024-03-29T05:01:21.612 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.ResizeLocksImage 2024-03-29T05:01:21.638 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.ResizeLocksImage (26 ms) 2024-03-29T05:01:21.638 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.ResizeFailsToLockImage 2024-03-29T05:01:21.656 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.ResizeFailsToLockImage (18 ms) 2024-03-29T05:01:21.656 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.SnapCreateLocksImage 2024-03-29T05:01:21.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:21 smithi187 ceph-mon[93536]: pgmap v455: 105 pgs: 32 unknown, 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T05:01:21.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:21 smithi187 ceph-mon[93536]: osdmap e528: 8 total, 8 up, 8 in 2024-03-29T05:01:21.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:21 smithi187 ceph-mon[93536]: failed to encode map e528 with expected crc 2024-03-29T05:01:21.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:21 smithi187 ceph-mon[93536]: failed to encode map e528 with expected crc 2024-03-29T05:01:21.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:21 smithi181 ceph-mon[106466]: pgmap v455: 105 pgs: 32 unknown, 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T05:01:21.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:21 smithi181 ceph-mon[106466]: osdmap e528: 8 total, 8 up, 8 in 2024-03-29T05:01:21.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:21 smithi181 ceph-mon[106466]: failed to encode map e528 with expected crc 2024-03-29T05:01:21.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:21 smithi181 ceph-mon[106466]: failed to encode map e528 with expected crc 2024-03-29T05:01:21.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:21 smithi181 ceph-mon[101970]: pgmap v455: 105 pgs: 32 unknown, 2 active+undersized+degraded, 16 active+undersized, 55 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-03-29T05:01:21.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:21 smithi181 ceph-mon[101970]: osdmap e528: 8 total, 8 up, 8 in 2024-03-29T05:01:21.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:21 smithi181 ceph-mon[101970]: failed to encode map e528 with expected crc 2024-03-29T05:01:21.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:21 smithi181 ceph-mon[101970]: failed to encode map e528 with expected crc 2024-03-29T05:01:21.955 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:01:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T05:01:21.729+0000 7f67753a3700 -1 mgr.server reply reply (11) Resource temporarily unavailable 22 pgs have unknown state; cannot draw any conclusions 2024-03-29T05:01:21.955 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:01:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T05:01:21.729+0000 7f67753a3700 -1 mgr.server reply reply (16) Device or resource busy 22 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-03-29T05:01:22.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: failed to encode map e529 with expected crc 2024-03-29T05:01:22.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: osdmap e529: 8 total, 8 up, 8 in 2024-03-29T05:01:22.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: failed to encode map e529 with expected crc 2024-03-29T05:01:22.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: failed to encode map e529 with expected crc 2024-03-29T05:01:22.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: failed to encode map e529 with expected crc 2024-03-29T05:01:22.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:22.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:22.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:01:22.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:01:22.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: pgmap v458: 105 pgs: 22 unknown, 1 active+undersized+degraded, 10 active+undersized, 72 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 626 B/s wr, 15 op/s; 2/78 objects degraded (2.564%) 2024-03-29T05:01:22.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: pgmap v459: 105 pgs: 22 unknown, 1 active+undersized+degraded, 10 active+undersized, 72 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 662 B/s wr, 16 op/s; 2/78 objects degraded (2.564%) 2024-03-29T05:01:22.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:22.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:01:22.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:22.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:22.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:22.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T05:01:22.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T05:01:22.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: Upgrade: 22 pgs have unknown state; cannot draw any conclusions 2024-03-29T05:01:22.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:22 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/645557646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:22.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: failed to encode map e529 with expected crc 2024-03-29T05:01:22.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: osdmap e529: 8 total, 8 up, 8 in 2024-03-29T05:01:22.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: failed to encode map e529 with expected crc 2024-03-29T05:01:22.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: failed to encode map e529 with expected crc 2024-03-29T05:01:22.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: failed to encode map e529 with expected crc 2024-03-29T05:01:22.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:22.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:22.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:01:22.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:01:22.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: pgmap v458: 105 pgs: 22 unknown, 1 active+undersized+degraded, 10 active+undersized, 72 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 626 B/s wr, 15 op/s; 2/78 objects degraded (2.564%) 2024-03-29T05:01:22.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: pgmap v459: 105 pgs: 22 unknown, 1 active+undersized+degraded, 10 active+undersized, 72 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 662 B/s wr, 16 op/s; 2/78 objects degraded (2.564%) 2024-03-29T05:01:22.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:22.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:01:22.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:22.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:22.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:22.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T05:01:22.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T05:01:22.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: Upgrade: 22 pgs have unknown state; cannot draw any conclusions 2024-03-29T05:01:22.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/645557646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:22.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: failed to encode map e529 with expected crc 2024-03-29T05:01:22.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: osdmap e529: 8 total, 8 up, 8 in 2024-03-29T05:01:22.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: failed to encode map e529 with expected crc 2024-03-29T05:01:22.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: failed to encode map e529 with expected crc 2024-03-29T05:01:22.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: failed to encode map e529 with expected crc 2024-03-29T05:01:22.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:22.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:22.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:01:22.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:01:22.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: pgmap v458: 105 pgs: 22 unknown, 1 active+undersized+degraded, 10 active+undersized, 72 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 626 B/s wr, 15 op/s; 2/78 objects degraded (2.564%) 2024-03-29T05:01:22.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: pgmap v459: 105 pgs: 22 unknown, 1 active+undersized+degraded, 10 active+undersized, 72 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 662 B/s wr, 16 op/s; 2/78 objects degraded (2.564%) 2024-03-29T05:01:22.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:22.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:01:22.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:22.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:22.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:22.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T05:01:22.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T05:01:22.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: Upgrade: 22 pgs have unknown state; cannot draw any conclusions 2024-03-29T05:01:22.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:22 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/645557646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:23.623 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.SnapCreateLocksImage (1966 ms) 2024-03-29T05:01:23.623 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.SnapCreateFailsToLockImage 2024-03-29T05:01:23.649 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.SnapCreateFailsToLockImage (27 ms) 2024-03-29T05:01:23.649 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.SnapRollbackLocksImage 2024-03-29T05:01:23.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:23 smithi187 ceph-mon[93536]: osdmap e530: 8 total, 8 up, 8 in 2024-03-29T05:01:23.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:23 smithi187 ceph-mon[93536]: failed to encode map e530 with expected crc 2024-03-29T05:01:23.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:23 smithi187 ceph-mon[93536]: failed to encode map e530 with expected crc 2024-03-29T05:01:23.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:23 smithi181 ceph-mon[106466]: osdmap e530: 8 total, 8 up, 8 in 2024-03-29T05:01:23.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:23 smithi181 ceph-mon[106466]: failed to encode map e530 with expected crc 2024-03-29T05:01:23.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:23 smithi181 ceph-mon[106466]: failed to encode map e530 with expected crc 2024-03-29T05:01:23.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:23 smithi181 ceph-mon[101970]: osdmap e530: 8 total, 8 up, 8 in 2024-03-29T05:01:23.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:23 smithi181 ceph-mon[101970]: failed to encode map e530 with expected crc 2024-03-29T05:01:23.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:23 smithi181 ceph-mon[101970]: failed to encode map e530 with expected crc 2024-03-29T05:01:24.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:24 smithi187 ceph-mon[93536]: pgmap v461: 105 pgs: 105 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 4.0 KiB/s wr, 55 op/s 2024-03-29T05:01:24.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:24 smithi187 ceph-mon[93536]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/78 objects degraded (2.564%), 1 pg degraded) 2024-03-29T05:01:24.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:24 smithi187 ceph-mon[93536]: failed to encode map e531 with expected crc 2024-03-29T05:01:24.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:24 smithi187 ceph-mon[93536]: failed to encode map e531 with expected crc 2024-03-29T05:01:24.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:24 smithi187 ceph-mon[93536]: osdmap e531: 8 total, 8 up, 8 in 2024-03-29T05:01:24.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:24 smithi187 ceph-mon[93536]: failed to encode map e531 with expected crc 2024-03-29T05:01:24.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:24 smithi187 ceph-mon[93536]: failed to encode map e531 with expected crc 2024-03-29T05:01:24.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:24 smithi187 ceph-mon[93536]: failed to encode map e531 with expected crc 2024-03-29T05:01:24.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:24 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/132368466' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:24.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[101970]: pgmap v461: 105 pgs: 105 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 4.0 KiB/s wr, 55 op/s 2024-03-29T05:01:24.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[101970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/78 objects degraded (2.564%), 1 pg degraded) 2024-03-29T05:01:24.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[101970]: failed to encode map e531 with expected crc 2024-03-29T05:01:24.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[101970]: failed to encode map e531 with expected crc 2024-03-29T05:01:24.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[101970]: osdmap e531: 8 total, 8 up, 8 in 2024-03-29T05:01:24.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[101970]: failed to encode map e531 with expected crc 2024-03-29T05:01:24.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[101970]: failed to encode map e531 with expected crc 2024-03-29T05:01:24.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[101970]: failed to encode map e531 with expected crc 2024-03-29T05:01:24.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/132368466' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:24.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[106466]: pgmap v461: 105 pgs: 105 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 4.0 KiB/s wr, 55 op/s 2024-03-29T05:01:24.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[106466]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/78 objects degraded (2.564%), 1 pg degraded) 2024-03-29T05:01:24.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[106466]: failed to encode map e531 with expected crc 2024-03-29T05:01:24.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[106466]: failed to encode map e531 with expected crc 2024-03-29T05:01:24.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[106466]: osdmap e531: 8 total, 8 up, 8 in 2024-03-29T05:01:24.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[106466]: failed to encode map e531 with expected crc 2024-03-29T05:01:24.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[106466]: failed to encode map e531 with expected crc 2024-03-29T05:01:24.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[106466]: failed to encode map e531 with expected crc 2024-03-29T05:01:24.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:24 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/132368466' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:25.631 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.SnapRollbackLocksImage (1981 ms) 2024-03-29T05:01:25.631 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.SnapRollbackFailsToLockImage 2024-03-29T05:01:25.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:25 smithi187 ceph-mon[93536]: failed to encode map e532 with expected crc 2024-03-29T05:01:25.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:25 smithi187 ceph-mon[93536]: osdmap e532: 8 total, 8 up, 8 in 2024-03-29T05:01:25.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:25 smithi187 ceph-mon[93536]: failed to encode map e532 with expected crc 2024-03-29T05:01:25.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:25 smithi187 ceph-mon[93536]: failed to encode map e532 with expected crc 2024-03-29T05:01:25.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:25 smithi181 ceph-mon[106466]: failed to encode map e532 with expected crc 2024-03-29T05:01:25.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:25 smithi181 ceph-mon[106466]: osdmap e532: 8 total, 8 up, 8 in 2024-03-29T05:01:25.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:25 smithi181 ceph-mon[106466]: failed to encode map e532 with expected crc 2024-03-29T05:01:25.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:25 smithi181 ceph-mon[106466]: failed to encode map e532 with expected crc 2024-03-29T05:01:25.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:25 smithi181 ceph-mon[101970]: failed to encode map e532 with expected crc 2024-03-29T05:01:25.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:25 smithi181 ceph-mon[101970]: osdmap e532: 8 total, 8 up, 8 in 2024-03-29T05:01:25.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:25 smithi181 ceph-mon[101970]: failed to encode map e532 with expected crc 2024-03-29T05:01:25.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:25 smithi181 ceph-mon[101970]: failed to encode map e532 with expected crc 2024-03-29T05:01:26.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:26 smithi187 ceph-mon[93536]: pgmap v464: 105 pgs: 105 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.7 KiB/s wr, 32 op/s 2024-03-29T05:01:26.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:26 smithi187 ceph-mon[93536]: failed to encode map e533 with expected crc 2024-03-29T05:01:26.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:26 smithi187 ceph-mon[93536]: failed to encode map e533 with expected crc 2024-03-29T05:01:26.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:26 smithi187 ceph-mon[93536]: osdmap e533: 8 total, 8 up, 8 in 2024-03-29T05:01:26.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:26 smithi187 ceph-mon[93536]: failed to encode map e533 with expected crc 2024-03-29T05:01:26.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:26 smithi187 ceph-mon[93536]: failed to encode map e533 with expected crc 2024-03-29T05:01:26.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:26 smithi187 ceph-mon[93536]: failed to encode map e533 with expected crc 2024-03-29T05:01:26.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:26.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:01:26.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:26 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/436840783' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:26.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[101970]: pgmap v464: 105 pgs: 105 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.7 KiB/s wr, 32 op/s 2024-03-29T05:01:26.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[101970]: failed to encode map e533 with expected crc 2024-03-29T05:01:26.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[101970]: failed to encode map e533 with expected crc 2024-03-29T05:01:26.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[101970]: osdmap e533: 8 total, 8 up, 8 in 2024-03-29T05:01:26.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[101970]: failed to encode map e533 with expected crc 2024-03-29T05:01:26.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[101970]: failed to encode map e533 with expected crc 2024-03-29T05:01:26.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[101970]: failed to encode map e533 with expected crc 2024-03-29T05:01:26.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:26.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:01:26.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/436840783' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:26.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[106466]: pgmap v464: 105 pgs: 105 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.7 KiB/s wr, 32 op/s 2024-03-29T05:01:26.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[106466]: failed to encode map e533 with expected crc 2024-03-29T05:01:26.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[106466]: failed to encode map e533 with expected crc 2024-03-29T05:01:26.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[106466]: osdmap e533: 8 total, 8 up, 8 in 2024-03-29T05:01:26.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[106466]: failed to encode map e533 with expected crc 2024-03-29T05:01:26.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[106466]: failed to encode map e533 with expected crc 2024-03-29T05:01:26.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[106466]: failed to encode map e533 with expected crc 2024-03-29T05:01:26.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:26.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:01:26.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:26 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/436840783' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:27.642 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.SnapRollbackFailsToLockImage (2011 ms) 2024-03-29T05:01:27.642 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.SnapSetReleasesLock 2024-03-29T05:01:27.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:27 smithi187 ceph-mon[93536]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:27 smithi187 ceph-mon[93536]: osdmap e534: 8 total, 8 up, 8 in 2024-03-29T05:01:27.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:27 smithi187 ceph-mon[93536]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:27 smithi187 ceph-mon[93536]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:27 smithi187 ceph-mon[93536]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:27 smithi187 ceph-mon[93536]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:27 smithi187 ceph-mon[93536]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:27 smithi181 ceph-mon[106466]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:27 smithi181 ceph-mon[106466]: osdmap e534: 8 total, 8 up, 8 in 2024-03-29T05:01:27.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:27 smithi181 ceph-mon[106466]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:27 smithi181 ceph-mon[106466]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:27 smithi181 ceph-mon[106466]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:27 smithi181 ceph-mon[106466]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:27 smithi181 ceph-mon[106466]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:27 smithi181 ceph-mon[101970]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:27 smithi181 ceph-mon[101970]: osdmap e534: 8 total, 8 up, 8 in 2024-03-29T05:01:27.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:27 smithi181 ceph-mon[101970]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:27 smithi181 ceph-mon[101970]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:27 smithi181 ceph-mon[101970]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:27 smithi181 ceph-mon[101970]: failed to encode map e534 with expected crc 2024-03-29T05:01:27.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:27 smithi181 ceph-mon[101970]: failed to encode map e534 with expected crc 2024-03-29T05:01:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:28 smithi187 ceph-mon[93536]: pgmap v467: 105 pgs: 105 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 11 KiB/s wr, 128 op/s 2024-03-29T05:01:28.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:28 smithi187 ceph-mon[93536]: failed to encode map e535 with expected crc 2024-03-29T05:01:28.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:28 smithi187 ceph-mon[93536]: osdmap e535: 8 total, 8 up, 8 in 2024-03-29T05:01:28.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:28 smithi187 ceph-mon[93536]: failed to encode map e535 with expected crc 2024-03-29T05:01:28.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:28 smithi187 ceph-mon[93536]: failed to encode map e535 with expected crc 2024-03-29T05:01:28.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:28 smithi187 ceph-mon[93536]: failed to encode map e535 with expected crc 2024-03-29T05:01:28.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:28 smithi187 ceph-mon[93536]: failed to encode map e535 with expected crc 2024-03-29T05:01:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:28 smithi181 ceph-mon[101970]: pgmap v467: 105 pgs: 105 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 11 KiB/s wr, 128 op/s 2024-03-29T05:01:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:28 smithi181 ceph-mon[101970]: failed to encode map e535 with expected crc 2024-03-29T05:01:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:28 smithi181 ceph-mon[101970]: osdmap e535: 8 total, 8 up, 8 in 2024-03-29T05:01:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:28 smithi181 ceph-mon[101970]: failed to encode map e535 with expected crc 2024-03-29T05:01:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:28 smithi181 ceph-mon[101970]: failed to encode map e535 with expected crc 2024-03-29T05:01:28.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:28 smithi181 ceph-mon[101970]: failed to encode map e535 with expected crc 2024-03-29T05:01:28.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:28 smithi181 ceph-mon[101970]: failed to encode map e535 with expected crc 2024-03-29T05:01:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:28 smithi181 ceph-mon[106466]: pgmap v467: 105 pgs: 105 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 11 KiB/s wr, 128 op/s 2024-03-29T05:01:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:28 smithi181 ceph-mon[106466]: failed to encode map e535 with expected crc 2024-03-29T05:01:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:28 smithi181 ceph-mon[106466]: osdmap e535: 8 total, 8 up, 8 in 2024-03-29T05:01:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:28 smithi181 ceph-mon[106466]: failed to encode map e535 with expected crc 2024-03-29T05:01:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:28 smithi181 ceph-mon[106466]: failed to encode map e535 with expected crc 2024-03-29T05:01:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:28 smithi181 ceph-mon[106466]: failed to encode map e535 with expected crc 2024-03-29T05:01:28.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:28 smithi181 ceph-mon[106466]: failed to encode map e535 with expected crc 2024-03-29T05:01:29.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:01:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:01:28] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2024-03-29T05:01:29.659 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.SnapSetReleasesLock (2017 ms) 2024-03-29T05:01:29.659 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.FlattenLocksImage 2024-03-29T05:01:29.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:29 smithi181 ceph-mon[106466]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:29 smithi181 ceph-mon[106466]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:29 smithi181 ceph-mon[106466]: osdmap e536: 8 total, 8 up, 8 in 2024-03-29T05:01:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:29 smithi181 ceph-mon[106466]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:29 smithi181 ceph-mon[106466]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:29 smithi181 ceph-mon[106466]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:29 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2602888508' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:29 smithi181 ceph-mon[106466]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:29 smithi181 ceph-mon[101970]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:29 smithi181 ceph-mon[101970]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:29 smithi181 ceph-mon[101970]: osdmap e536: 8 total, 8 up, 8 in 2024-03-29T05:01:29.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:29 smithi181 ceph-mon[101970]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:29 smithi181 ceph-mon[101970]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:29 smithi181 ceph-mon[101970]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:29 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2602888508' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:29.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:29 smithi181 ceph-mon[101970]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.991 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:29 smithi187 ceph-mon[93536]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.991 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:29 smithi187 ceph-mon[93536]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.991 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:29 smithi187 ceph-mon[93536]: osdmap e536: 8 total, 8 up, 8 in 2024-03-29T05:01:29.991 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:29 smithi187 ceph-mon[93536]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.991 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:29 smithi187 ceph-mon[93536]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.991 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:29 smithi187 ceph-mon[93536]: failed to encode map e536 with expected crc 2024-03-29T05:01:29.991 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:29 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2602888508' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:29.992 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:29 smithi187 ceph-mon[93536]: failed to encode map e536 with expected crc 2024-03-29T05:01:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:30 smithi181 ceph-mon[106466]: pgmap v470: 105 pgs: 105 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 11 KiB/s wr, 128 op/s 2024-03-29T05:01:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:30 smithi181 ceph-mon[106466]: failed to encode map e537 with expected crc 2024-03-29T05:01:30.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:30 smithi181 ceph-mon[106466]: osdmap e537: 8 total, 8 up, 8 in 2024-03-29T05:01:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:30 smithi181 ceph-mon[106466]: failed to encode map e537 with expected crc 2024-03-29T05:01:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:30 smithi181 ceph-mon[106466]: failed to encode map e537 with expected crc 2024-03-29T05:01:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:30 smithi181 ceph-mon[106466]: failed to encode map e537 with expected crc 2024-03-29T05:01:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:30 smithi181 ceph-mon[106466]: failed to encode map e537 with expected crc 2024-03-29T05:01:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:30 smithi181 ceph-mon[101970]: pgmap v470: 105 pgs: 105 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 11 KiB/s wr, 128 op/s 2024-03-29T05:01:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:30 smithi181 ceph-mon[101970]: failed to encode map e537 with expected crc 2024-03-29T05:01:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:30 smithi181 ceph-mon[101970]: osdmap e537: 8 total, 8 up, 8 in 2024-03-29T05:01:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:30 smithi181 ceph-mon[101970]: failed to encode map e537 with expected crc 2024-03-29T05:01:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:30 smithi181 ceph-mon[101970]: failed to encode map e537 with expected crc 2024-03-29T05:01:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:30 smithi181 ceph-mon[101970]: failed to encode map e537 with expected crc 2024-03-29T05:01:30.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:30 smithi181 ceph-mon[101970]: failed to encode map e537 with expected crc 2024-03-29T05:01:31.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:30 smithi187 ceph-mon[93536]: pgmap v470: 105 pgs: 105 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 11 KiB/s wr, 128 op/s 2024-03-29T05:01:31.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:30 smithi187 ceph-mon[93536]: failed to encode map e537 with expected crc 2024-03-29T05:01:31.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:30 smithi187 ceph-mon[93536]: osdmap e537: 8 total, 8 up, 8 in 2024-03-29T05:01:31.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:30 smithi187 ceph-mon[93536]: failed to encode map e537 with expected crc 2024-03-29T05:01:31.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:30 smithi187 ceph-mon[93536]: failed to encode map e537 with expected crc 2024-03-29T05:01:31.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:30 smithi187 ceph-mon[93536]: failed to encode map e537 with expected crc 2024-03-29T05:01:31.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:30 smithi187 ceph-mon[93536]: failed to encode map e537 with expected crc 2024-03-29T05:01:31.674 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.FlattenLocksImage (2014 ms) 2024-03-29T05:01:31.674 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.FlattenFailsToLockImage 2024-03-29T05:01:31.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:31 smithi181 ceph-mon[106466]: failed to encode map e538 with expected crc 2024-03-29T05:01:31.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:31 smithi181 ceph-mon[106466]: osdmap e538: 8 total, 8 up, 8 in 2024-03-29T05:01:31.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:31 smithi181 ceph-mon[106466]: failed to encode map e538 with expected crc 2024-03-29T05:01:31.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:31 smithi181 ceph-mon[106466]: failed to encode map e538 with expected crc 2024-03-29T05:01:31.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:31 smithi181 ceph-mon[106466]: failed to encode map e538 with expected crc 2024-03-29T05:01:31.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:31 smithi181 ceph-mon[106466]: failed to encode map e538 with expected crc 2024-03-29T05:01:31.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:31 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/767366928' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:31.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:31 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:31.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:31 smithi181 ceph-mon[101970]: failed to encode map e538 with expected crc 2024-03-29T05:01:31.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:31 smithi181 ceph-mon[101970]: osdmap e538: 8 total, 8 up, 8 in 2024-03-29T05:01:31.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:31 smithi181 ceph-mon[101970]: failed to encode map e538 with expected crc 2024-03-29T05:01:31.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:31 smithi181 ceph-mon[101970]: failed to encode map e538 with expected crc 2024-03-29T05:01:31.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:31 smithi181 ceph-mon[101970]: failed to encode map e538 with expected crc 2024-03-29T05:01:31.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:31 smithi181 ceph-mon[101970]: failed to encode map e538 with expected crc 2024-03-29T05:01:31.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:31 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/767366928' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:31.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:31 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:32.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:31 smithi187 ceph-mon[93536]: failed to encode map e538 with expected crc 2024-03-29T05:01:32.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:31 smithi187 ceph-mon[93536]: osdmap e538: 8 total, 8 up, 8 in 2024-03-29T05:01:32.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:31 smithi187 ceph-mon[93536]: failed to encode map e538 with expected crc 2024-03-29T05:01:32.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:31 smithi187 ceph-mon[93536]: failed to encode map e538 with expected crc 2024-03-29T05:01:32.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:31 smithi187 ceph-mon[93536]: failed to encode map e538 with expected crc 2024-03-29T05:01:32.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:31 smithi187 ceph-mon[93536]: failed to encode map e538 with expected crc 2024-03-29T05:01:32.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:31 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/767366928' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:32.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:31 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:32.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:32 smithi181 ceph-mon[106466]: pgmap v473: 105 pgs: 105 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 111 KiB/s rd, 7.5 KiB/s wr, 147 op/s 2024-03-29T05:01:32.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:32 smithi181 ceph-mon[106466]: failed to encode map e539 with expected crc 2024-03-29T05:01:32.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:32 smithi181 ceph-mon[106466]: failed to encode map e539 with expected crc 2024-03-29T05:01:32.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:32 smithi181 ceph-mon[106466]: osdmap e539: 8 total, 8 up, 8 in 2024-03-29T05:01:32.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:32 smithi181 ceph-mon[106466]: failed to encode map e539 with expected crc 2024-03-29T05:01:32.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:32 smithi181 ceph-mon[106466]: osdmap e540: 8 total, 8 up, 8 in 2024-03-29T05:01:32.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:32 smithi181 ceph-mon[106466]: failed to encode map e540 with expected crc 2024-03-29T05:01:32.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:32 smithi181 ceph-mon[101970]: pgmap v473: 105 pgs: 105 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 111 KiB/s rd, 7.5 KiB/s wr, 147 op/s 2024-03-29T05:01:32.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:32 smithi181 ceph-mon[101970]: failed to encode map e539 with expected crc 2024-03-29T05:01:32.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:32 smithi181 ceph-mon[101970]: failed to encode map e539 with expected crc 2024-03-29T05:01:32.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:32 smithi181 ceph-mon[101970]: osdmap e539: 8 total, 8 up, 8 in 2024-03-29T05:01:32.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:32 smithi181 ceph-mon[101970]: failed to encode map e539 with expected crc 2024-03-29T05:01:32.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:32 smithi181 ceph-mon[101970]: osdmap e540: 8 total, 8 up, 8 in 2024-03-29T05:01:32.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:32 smithi181 ceph-mon[101970]: failed to encode map e540 with expected crc 2024-03-29T05:01:33.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:32 smithi187 ceph-mon[93536]: pgmap v473: 105 pgs: 105 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 111 KiB/s rd, 7.5 KiB/s wr, 147 op/s 2024-03-29T05:01:33.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:32 smithi187 ceph-mon[93536]: failed to encode map e539 with expected crc 2024-03-29T05:01:33.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:32 smithi187 ceph-mon[93536]: failed to encode map e539 with expected crc 2024-03-29T05:01:33.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:32 smithi187 ceph-mon[93536]: osdmap e539: 8 total, 8 up, 8 in 2024-03-29T05:01:33.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:32 smithi187 ceph-mon[93536]: failed to encode map e539 with expected crc 2024-03-29T05:01:33.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:32 smithi187 ceph-mon[93536]: osdmap e540: 8 total, 8 up, 8 in 2024-03-29T05:01:33.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:32 smithi187 ceph-mon[93536]: failed to encode map e540 with expected crc 2024-03-29T05:01:33.685 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.FlattenFailsToLockImage (2011 ms) 2024-03-29T05:01:33.685 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklisted 2024-03-29T05:01:33.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:33 smithi181 ceph-mon[101970]: failed to encode map e540 with expected crc 2024-03-29T05:01:33.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:33 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3314516484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:33.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:33 smithi181 ceph-mon[106466]: failed to encode map e540 with expected crc 2024-03-29T05:01:33.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:33 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3314516484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:34.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:33 smithi187 ceph-mon[93536]: failed to encode map e540 with expected crc 2024-03-29T05:01:34.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:33 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3314516484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:34.682 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklisted (998 ms) 2024-03-29T05:01:34.683 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklistedWatch 2024-03-29T05:01:34.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:34 smithi181 ceph-mon[101970]: pgmap v476: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 9.5 KiB/s wr, 182 op/s 2024-03-29T05:01:34.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:34 smithi181 ceph-mon[101970]: failed to encode map e541 with expected crc 2024-03-29T05:01:34.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:34 smithi181 ceph-mon[101970]: osdmap e541: 8 total, 8 up, 8 in 2024-03-29T05:01:34.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:34 smithi181 ceph-mon[101970]: failed to encode map e541 with expected crc 2024-03-29T05:01:34.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:34 smithi181 ceph-mon[101970]: failed to encode map e541 with expected crc 2024-03-29T05:01:34.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:34 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2023235' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2023235"}]: dispatch 2024-03-29T05:01:34.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:34 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2023235"}]: dispatch 2024-03-29T05:01:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:34 smithi181 ceph-mon[106466]: pgmap v476: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 9.5 KiB/s wr, 182 op/s 2024-03-29T05:01:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:34 smithi181 ceph-mon[106466]: failed to encode map e541 with expected crc 2024-03-29T05:01:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:34 smithi181 ceph-mon[106466]: osdmap e541: 8 total, 8 up, 8 in 2024-03-29T05:01:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:34 smithi181 ceph-mon[106466]: failed to encode map e541 with expected crc 2024-03-29T05:01:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:34 smithi181 ceph-mon[106466]: failed to encode map e541 with expected crc 2024-03-29T05:01:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:34 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2023235' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2023235"}]: dispatch 2024-03-29T05:01:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:34 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2023235"}]: dispatch 2024-03-29T05:01:35.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:34 smithi187 ceph-mon[93536]: pgmap v476: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 9.5 KiB/s wr, 182 op/s 2024-03-29T05:01:35.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:34 smithi187 ceph-mon[93536]: failed to encode map e541 with expected crc 2024-03-29T05:01:35.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:34 smithi187 ceph-mon[93536]: osdmap e541: 8 total, 8 up, 8 in 2024-03-29T05:01:35.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:34 smithi187 ceph-mon[93536]: failed to encode map e541 with expected crc 2024-03-29T05:01:35.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:34 smithi187 ceph-mon[93536]: failed to encode map e541 with expected crc 2024-03-29T05:01:35.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:34 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2023235' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2023235"}]: dispatch 2024-03-29T05:01:35.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:34 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2023235"}]: dispatch 2024-03-29T05:01:35.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:35 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2023235"}]': finished 2024-03-29T05:01:35.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:35 smithi181 ceph-mon[101970]: failed to encode map e542 with expected crc 2024-03-29T05:01:35.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:35 smithi181 ceph-mon[101970]: osdmap e542: 8 total, 8 up, 8 in 2024-03-29T05:01:35.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:35 smithi181 ceph-mon[101970]: failed to encode map e542 with expected crc 2024-03-29T05:01:35.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:35 smithi181 ceph-mon[101970]: failed to encode map e542 with expected crc 2024-03-29T05:01:35.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:35 smithi181 ceph-mon[101970]: failed to encode map e542 with expected crc 2024-03-29T05:01:35.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:35 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3116926796' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/3116926796"}]: dispatch 2024-03-29T05:01:35.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:35 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/3116926796"}]: dispatch 2024-03-29T05:01:35.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:35 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2023235"}]': finished 2024-03-29T05:01:35.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:35 smithi181 ceph-mon[106466]: failed to encode map e542 with expected crc 2024-03-29T05:01:35.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:35 smithi181 ceph-mon[106466]: osdmap e542: 8 total, 8 up, 8 in 2024-03-29T05:01:35.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:35 smithi181 ceph-mon[106466]: failed to encode map e542 with expected crc 2024-03-29T05:01:35.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:35 smithi181 ceph-mon[106466]: failed to encode map e542 with expected crc 2024-03-29T05:01:35.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:35 smithi181 ceph-mon[106466]: failed to encode map e542 with expected crc 2024-03-29T05:01:35.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:35 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3116926796' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/3116926796"}]: dispatch 2024-03-29T05:01:35.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:35 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/3116926796"}]: dispatch 2024-03-29T05:01:36.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:35 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2023235"}]': finished 2024-03-29T05:01:36.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:35 smithi187 ceph-mon[93536]: failed to encode map e542 with expected crc 2024-03-29T05:01:36.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:35 smithi187 ceph-mon[93536]: osdmap e542: 8 total, 8 up, 8 in 2024-03-29T05:01:36.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:35 smithi187 ceph-mon[93536]: failed to encode map e542 with expected crc 2024-03-29T05:01:36.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:35 smithi187 ceph-mon[93536]: failed to encode map e542 with expected crc 2024-03-29T05:01:36.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:35 smithi187 ceph-mon[93536]: failed to encode map e542 with expected crc 2024-03-29T05:01:36.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:35 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3116926796' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/3116926796"}]: dispatch 2024-03-29T05:01:36.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:35 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/3116926796"}]: dispatch 2024-03-29T05:01:36.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:36 smithi181 ceph-mon[106466]: pgmap v479: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 KiB/s wr, 34 op/s 2024-03-29T05:01:36.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:36 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/3116926796"}]': finished 2024-03-29T05:01:36.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:36 smithi181 ceph-mon[106466]: failed to encode map e543 with expected crc 2024-03-29T05:01:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:36 smithi181 ceph-mon[106466]: osdmap e543: 8 total, 8 up, 8 in 2024-03-29T05:01:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:36 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/126745292' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:36 smithi181 ceph-mon[106466]: failed to encode map e543 with expected crc 2024-03-29T05:01:36.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:36 smithi181 ceph-mon[106466]: failed to encode map e543 with expected crc 2024-03-29T05:01:36.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:36 smithi181 ceph-mon[101970]: pgmap v479: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 KiB/s wr, 34 op/s 2024-03-29T05:01:36.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:36 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/3116926796"}]': finished 2024-03-29T05:01:36.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:36 smithi181 ceph-mon[101970]: failed to encode map e543 with expected crc 2024-03-29T05:01:36.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:36 smithi181 ceph-mon[101970]: osdmap e543: 8 total, 8 up, 8 in 2024-03-29T05:01:36.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:36 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/126745292' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:36.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:36 smithi181 ceph-mon[101970]: failed to encode map e543 with expected crc 2024-03-29T05:01:36.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:36 smithi181 ceph-mon[101970]: failed to encode map e543 with expected crc 2024-03-29T05:01:37.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:36 smithi187 ceph-mon[93536]: pgmap v479: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 KiB/s wr, 34 op/s 2024-03-29T05:01:37.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:36 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/3116926796"}]': finished 2024-03-29T05:01:37.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:36 smithi187 ceph-mon[93536]: failed to encode map e543 with expected crc 2024-03-29T05:01:37.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:36 smithi187 ceph-mon[93536]: osdmap e543: 8 total, 8 up, 8 in 2024-03-29T05:01:37.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:36 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/126745292' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:37.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:36 smithi187 ceph-mon[93536]: failed to encode map e543 with expected crc 2024-03-29T05:01:37.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:36 smithi187 ceph-mon[93536]: failed to encode map e543 with expected crc 2024-03-29T05:01:37.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:37 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T05:01:37.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:37 smithi181 ceph-mon[101970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T05:01:37.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:37 smithi181 ceph-mon[101970]: Upgrade: osd.3 is safe to restart 2024-03-29T05:01:37.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:37 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T05:01:37.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:37 smithi181 ceph-mon[106466]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T05:01:37.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:37 smithi181 ceph-mon[106466]: Upgrade: osd.3 is safe to restart 2024-03-29T05:01:38.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:37 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T05:01:38.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:37 smithi187 ceph-mon[93536]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T05:01:38.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:37 smithi187 ceph-mon[93536]: Upgrade: osd.3 is safe to restart 2024-03-29T05:01:38.939 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:38 smithi181 ceph-mon[101970]: pgmap v481: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 5.9 KiB/s wr, 116 op/s 2024-03-29T05:01:38.939 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:38 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1824173919' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:38.939 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:38.939 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T05:01:38.939 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:01:38.939 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:38 smithi181 ceph-mon[106466]: pgmap v481: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 5.9 KiB/s wr, 116 op/s 2024-03-29T05:01:38.939 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:38 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1824173919' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:38.939 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:38.939 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T05:01:38.940 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:01:39.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:38 smithi187 ceph-mon[93536]: pgmap v481: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 5.9 KiB/s wr, 116 op/s 2024-03-29T05:01:39.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:38 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1824173919' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:39.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:39.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T05:01:39.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:01:39.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:01:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:01:38] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.43.0" 2024-03-29T05:01:39.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:39 smithi187 ceph-mon[93536]: Upgrade: Updating osd.3 2024-03-29T05:01:39.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:39 smithi187 ceph-mon[93536]: Deploying daemon osd.3 on smithi181 2024-03-29T05:01:39.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:39 smithi181 ceph-mon[101970]: Upgrade: Updating osd.3 2024-03-29T05:01:39.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:39 smithi181 ceph-mon[101970]: Deploying daemon osd.3 on smithi181 2024-03-29T05:01:39.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:39 smithi181 ceph-mon[106466]: Upgrade: Updating osd.3 2024-03-29T05:01:39.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:39 smithi181 ceph-mon[106466]: Deploying daemon osd.3 on smithi181 2024-03-29T05:01:40.685 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklistedWatch (6002 ms) 2024-03-29T05:01:40.685 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.AioWriteRequestsLock 2024-03-29T05:01:40.717 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.AioWriteRequestsLock (33 ms) 2024-03-29T05:01:40.718 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.AioDiscardRequestsLock 2024-03-29T05:01:40.745 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.AioDiscardRequestsLock (28 ms) 2024-03-29T05:01:40.745 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.CancelAsyncResize 2024-03-29T05:01:40.789 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.CancelAsyncResize (44 ms) 2024-03-29T05:01:40.790 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.MultipleResize 2024-03-29T05:01:40.841 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.MultipleResize (51 ms) 2024-03-29T05:01:40.841 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.Metadata 2024-03-29T05:01:40.885 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.Metadata (44 ms) 2024-03-29T05:01:40.885 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.MetadataConfApply 2024-03-29T05:01:40.915 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.MetadataConfApply (30 ms) 2024-03-29T05:01:40.915 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.SnapshotCopyup 2024-03-29T05:01:40.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:40 smithi181 ceph-mon[101970]: pgmap v482: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 5.0 KiB/s wr, 98 op/s 2024-03-29T05:01:40.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:40 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/4047954721' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:40.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:40 smithi181 ceph-mon[106466]: pgmap v482: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 5.0 KiB/s wr, 98 op/s 2024-03-29T05:01:40.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:40 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/4047954721' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:41.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:40 smithi187 ceph-mon[93536]: pgmap v482: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 5.0 KiB/s wr, 98 op/s 2024-03-29T05:01:41.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:40 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/4047954721' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:41.954 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:41 smithi181 systemd[1]: Stopping Ceph osd.3 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:01:41.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:01:41.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:01:42.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:01:42.454 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3[64411]: 2024-03-29T05:01:42.090+0000 7f78c2a5d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T05:01:42.454 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3[64411]: 2024-03-29T05:01:42.090+0000 7f78c2a5d700 -1 osd.3 544 *** Got signal Terminated *** 2024-03-29T05:01:42.455 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3[64411]: 2024-03-29T05:01:42.090+0000 7f78c2a5d700 -1 osd.3 544 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T05:01:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[101970]: pgmap v483: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 87 KiB/s rd, 8.1 KiB/s wr, 119 op/s 2024-03-29T05:01:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[101970]: failed to encode map e544 with expected crc 2024-03-29T05:01:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[101970]: failed to encode map e544 with expected crc 2024-03-29T05:01:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[101970]: osdmap e544: 8 total, 8 up, 8 in 2024-03-29T05:01:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[101970]: failed to encode map e544 with expected crc 2024-03-29T05:01:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[101970]: failed to encode map e544 with expected crc 2024-03-29T05:01:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[101970]: failed to encode map e544 with expected crc 2024-03-29T05:01:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[101970]: osd.3 marked itself down and dead 2024-03-29T05:01:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[101970]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T05:01:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[101970]: failed to encode map e545 with expected crc 2024-03-29T05:01:42.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[101970]: osdmap e545: 8 total, 7 up, 8 in 2024-03-29T05:01:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[106466]: pgmap v483: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 87 KiB/s rd, 8.1 KiB/s wr, 119 op/s 2024-03-29T05:01:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[106466]: failed to encode map e544 with expected crc 2024-03-29T05:01:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[106466]: failed to encode map e544 with expected crc 2024-03-29T05:01:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[106466]: osdmap e544: 8 total, 8 up, 8 in 2024-03-29T05:01:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[106466]: failed to encode map e544 with expected crc 2024-03-29T05:01:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[106466]: failed to encode map e544 with expected crc 2024-03-29T05:01:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[106466]: failed to encode map e544 with expected crc 2024-03-29T05:01:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[106466]: osd.3 marked itself down and dead 2024-03-29T05:01:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[106466]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T05:01:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[106466]: failed to encode map e545 with expected crc 2024-03-29T05:01:42.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:42 smithi181 ceph-mon[106466]: osdmap e545: 8 total, 7 up, 8 in 2024-03-29T05:01:43.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:42 smithi187 ceph-mon[93536]: pgmap v483: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 87 KiB/s rd, 8.1 KiB/s wr, 119 op/s 2024-03-29T05:01:43.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:42 smithi187 ceph-mon[93536]: failed to encode map e544 with expected crc 2024-03-29T05:01:43.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:42 smithi187 ceph-mon[93536]: failed to encode map e544 with expected crc 2024-03-29T05:01:43.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:42 smithi187 ceph-mon[93536]: osdmap e544: 8 total, 8 up, 8 in 2024-03-29T05:01:43.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:42 smithi187 ceph-mon[93536]: failed to encode map e544 with expected crc 2024-03-29T05:01:43.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:42 smithi187 ceph-mon[93536]: failed to encode map e544 with expected crc 2024-03-29T05:01:43.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:42 smithi187 ceph-mon[93536]: failed to encode map e544 with expected crc 2024-03-29T05:01:43.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:42 smithi187 ceph-mon[93536]: osd.3 marked itself down and dead 2024-03-29T05:01:43.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:42 smithi187 ceph-mon[93536]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T05:01:43.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:42 smithi187 ceph-mon[93536]: failed to encode map e545 with expected crc 2024-03-29T05:01:43.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:42 smithi187 ceph-mon[93536]: osdmap e545: 8 total, 7 up, 8 in 2024-03-29T05:01:43.339 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:43 smithi181 podman[131898]: 2024-03-29 05:01:43.027138482 +0000 UTC m=+1.146572669 container died 6b0bf471cfcb77ade47bd42b1f34e1bb955ad604416460d4b0e97c842d740506 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-03-29T05:01:43.614 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:43 smithi181 podman[131898]: 2024-03-29 05:01:43.421777998 +0000 UTC m=+1.541212177 container cleanup 6b0bf471cfcb77ade47bd42b1f34e1bb955ad604416460d4b0e97c842d740506 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-03-29T05:01:43.615 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:43 smithi181 bash[131898]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3 2024-03-29T05:01:43.954 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:43 smithi181 podman[131957]: 2024-03-29 05:01:43.606752125 +0000 UTC m=+0.575751060 container remove 6b0bf471cfcb77ade47bd42b1f34e1bb955ad604416460d4b0e97c842d740506 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2) 2024-03-29T05:01:43.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:43 smithi181 ceph-mon[106466]: failed to encode map e545 with expected crc 2024-03-29T05:01:43.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:43 smithi181 ceph-mon[106466]: failed to encode map e545 with expected crc 2024-03-29T05:01:43.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:43 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/4263060657' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:43.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:43 smithi181 ceph-mon[106466]: failed to encode map e546 with expected crc 2024-03-29T05:01:43.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:43 smithi181 ceph-mon[106466]: failed to encode map e546 with expected crc 2024-03-29T05:01:43.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:43 smithi181 ceph-mon[106466]: osdmap e546: 8 total, 7 up, 8 in 2024-03-29T05:01:43.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:43 smithi181 ceph-mon[101970]: failed to encode map e545 with expected crc 2024-03-29T05:01:43.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:43 smithi181 ceph-mon[101970]: failed to encode map e545 with expected crc 2024-03-29T05:01:43.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:43 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/4263060657' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:43.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:43 smithi181 ceph-mon[101970]: failed to encode map e546 with expected crc 2024-03-29T05:01:43.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:43 smithi181 ceph-mon[101970]: failed to encode map e546 with expected crc 2024-03-29T05:01:43.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:43 smithi181 ceph-mon[101970]: osdmap e546: 8 total, 7 up, 8 in 2024-03-29T05:01:44.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:43 smithi187 ceph-mon[93536]: failed to encode map e545 with expected crc 2024-03-29T05:01:44.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:43 smithi187 ceph-mon[93536]: failed to encode map e545 with expected crc 2024-03-29T05:01:44.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:43 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/4263060657' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:44.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:43 smithi187 ceph-mon[93536]: failed to encode map e546 with expected crc 2024-03-29T05:01:44.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:43 smithi187 ceph-mon[93536]: failed to encode map e546 with expected crc 2024-03-29T05:01:44.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:43 smithi187 ceph-mon[93536]: osdmap e546: 8 total, 7 up, 8 in 2024-03-29T05:01:44.293 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:44 smithi181 podman[132052]: 2024-03-29 05:01:43.934609445 +0000 UTC m=+0.017864519 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:01:44.294 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:44 smithi181 podman[132052]: 2024-03-29 05:01:44.066336579 +0000 UTC m=+0.149591635 container create b916397cf79940df4509a4d8ffd083d1046ef8c2fade9358cc93e29bd415ad68 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-03-29T05:01:44.294 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:44 smithi181 podman[132052]: 2024-03-29 05:01:44.28574994 +0000 UTC m=+0.369005009 container init b916397cf79940df4509a4d8ffd083d1046ef8c2fade9358cc93e29bd415ad68 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-deactivate, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326) 2024-03-29T05:01:44.595 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:44 smithi181 podman[132052]: 2024-03-29 05:01:44.291551619 +0000 UTC m=+0.374806676 container start b916397cf79940df4509a4d8ffd083d1046ef8c2fade9358cc93e29bd415ad68 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-03-29T05:01:44.595 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:44 smithi181 podman[132052]: 2024-03-29 05:01:44.394161234 +0000 UTC m=+0.477416303 container attach b916397cf79940df4509a4d8ffd083d1046ef8c2fade9358cc93e29bd415ad68 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-03-29T05:01:44.606 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.SnapshotCopyup (3692 ms) 2024-03-29T05:01:44.607 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.SnapshotCopyupZeros 2024-03-29T05:01:44.915 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:44 smithi181 podman[132052]: 2024-03-29 05:01:44.587042231 +0000 UTC m=+0.670297278 container died b916397cf79940df4509a4d8ffd083d1046ef8c2fade9358cc93e29bd415ad68 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326) 2024-03-29T05:01:44.915 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:44 smithi181 ceph-mon[101970]: failed to encode map e546 with expected crc 2024-03-29T05:01:44.915 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:44 smithi181 ceph-mon[101970]: failed to encode map e546 with expected crc 2024-03-29T05:01:44.915 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:44 smithi181 ceph-mon[101970]: failed to encode map e546 with expected crc 2024-03-29T05:01:44.915 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:44 smithi181 ceph-mon[101970]: failed to encode map e546 with expected crc 2024-03-29T05:01:44.915 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:44 smithi181 ceph-mon[101970]: failed to encode map e546 with expected crc 2024-03-29T05:01:44.915 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:44 smithi181 ceph-mon[101970]: failed to encode map e546 with expected crc 2024-03-29T05:01:44.915 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:44 smithi181 ceph-mon[101970]: pgmap v487: 105 pgs: 19 stale+active+clean, 86 active+clean; 585 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 16 KiB/s wr, 133 op/s 2024-03-29T05:01:44.915 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:44 smithi181 ceph-mon[101970]: osdmap e547: 8 total, 7 up, 8 in 2024-03-29T05:01:44.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:44 smithi181 ceph-mon[106466]: failed to encode map e546 with expected crc 2024-03-29T05:01:44.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:44 smithi181 ceph-mon[106466]: failed to encode map e546 with expected crc 2024-03-29T05:01:44.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:44 smithi181 ceph-mon[106466]: failed to encode map e546 with expected crc 2024-03-29T05:01:44.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:44 smithi181 ceph-mon[106466]: failed to encode map e546 with expected crc 2024-03-29T05:01:44.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:44 smithi181 ceph-mon[106466]: failed to encode map e546 with expected crc 2024-03-29T05:01:44.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:44 smithi181 ceph-mon[106466]: failed to encode map e546 with expected crc 2024-03-29T05:01:44.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:44 smithi181 ceph-mon[106466]: pgmap v487: 105 pgs: 19 stale+active+clean, 86 active+clean; 585 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 16 KiB/s wr, 133 op/s 2024-03-29T05:01:44.916 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:44 smithi181 ceph-mon[106466]: osdmap e547: 8 total, 7 up, 8 in 2024-03-29T05:01:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:44 smithi187 ceph-mon[93536]: failed to encode map e546 with expected crc 2024-03-29T05:01:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:44 smithi187 ceph-mon[93536]: failed to encode map e546 with expected crc 2024-03-29T05:01:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:44 smithi187 ceph-mon[93536]: failed to encode map e546 with expected crc 2024-03-29T05:01:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:44 smithi187 ceph-mon[93536]: failed to encode map e546 with expected crc 2024-03-29T05:01:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:44 smithi187 ceph-mon[93536]: failed to encode map e546 with expected crc 2024-03-29T05:01:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:44 smithi187 ceph-mon[93536]: failed to encode map e546 with expected crc 2024-03-29T05:01:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:44 smithi187 ceph-mon[93536]: pgmap v487: 105 pgs: 19 stale+active+clean, 86 active+clean; 585 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 16 KiB/s wr, 133 op/s 2024-03-29T05:01:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:44 smithi187 ceph-mon[93536]: osdmap e547: 8 total, 7 up, 8 in 2024-03-29T05:01:45.638 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:45 smithi181 podman[132099]: 2024-03-29 05:01:45.198105816 +0000 UTC m=+0.604723062 container remove b916397cf79940df4509a4d8ffd083d1046ef8c2fade9358cc93e29bd415ad68 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux ) 2024-03-29T05:01:45.639 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:45 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.3.service: Deactivated successfully. 2024-03-29T05:01:45.639 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:45 smithi181 systemd[1]: Stopped Ceph osd.3 for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T05:01:45.639 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:45 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.3.service: Consumed 22.345s CPU time. 2024-03-29T05:01:45.639 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:45 smithi181 systemd[1]: Starting Ceph osd.3 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:01:45.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:45 smithi181 ceph-mon[101970]: failed to encode map e547 with expected crc 2024-03-29T05:01:45.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:45 smithi181 ceph-mon[101970]: failed to encode map e547 with expected crc 2024-03-29T05:01:45.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:45 smithi181 ceph-mon[101970]: failed to encode map e547 with expected crc 2024-03-29T05:01:45.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:45 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3924255906' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:45.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:45 smithi181 ceph-mon[101970]: osdmap e548: 8 total, 7 up, 8 in 2024-03-29T05:01:45.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:45 smithi181 ceph-mon[106466]: failed to encode map e547 with expected crc 2024-03-29T05:01:45.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:45 smithi181 ceph-mon[106466]: failed to encode map e547 with expected crc 2024-03-29T05:01:45.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:45 smithi181 ceph-mon[106466]: failed to encode map e547 with expected crc 2024-03-29T05:01:45.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:45 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3924255906' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:45.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:45 smithi181 ceph-mon[106466]: osdmap e548: 8 total, 7 up, 8 in 2024-03-29T05:01:45.953 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:45 smithi181 podman[132229]: 2024-03-29 05:01:45.533593372 +0000 UTC m=+0.019543337 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:01:45.953 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:45 smithi181 podman[132229]: 2024-03-29 05:01:45.648034253 +0000 UTC m=+0.133984202 container create c9d1e7c28f086a9bc40d0cf857e7333bbe2a12164f01f6a93c37a66af9227393 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T05:01:45.953 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:45 smithi181 podman[132229]: 2024-03-29 05:01:45.844241265 +0000 UTC m=+0.330191228 container init c9d1e7c28f086a9bc40d0cf857e7333bbe2a12164f01f6a93c37a66af9227393 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-03-29T05:01:45.953 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:45 smithi181 podman[132229]: 2024-03-29 05:01:45.849459425 +0000 UTC m=+0.335409380 container start c9d1e7c28f086a9bc40d0cf857e7333bbe2a12164f01f6a93c37a66af9227393 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-activate, ceph=True, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-8da2362, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T05:01:45.953 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:45 smithi181 podman[132229]: 2024-03-29 05:01:45.944117241 +0000 UTC m=+0.430067209 container attach c9d1e7c28f086a9bc40d0cf857e7333bbe2a12164f01f6a93c37a66af9227393 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-03-29T05:01:46.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:45 smithi187 ceph-mon[93536]: failed to encode map e547 with expected crc 2024-03-29T05:01:46.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:45 smithi187 ceph-mon[93536]: failed to encode map e547 with expected crc 2024-03-29T05:01:46.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:45 smithi187 ceph-mon[93536]: failed to encode map e547 with expected crc 2024-03-29T05:01:46.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:45 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3924255906' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:46.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:45 smithi187 ceph-mon[93536]: osdmap e548: 8 total, 7 up, 8 in 2024-03-29T05:01:46.549 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.SnapshotCopyupZeros (1942 ms) 2024-03-29T05:01:46.549 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.SnapshotCopyupZerosMigration 2024-03-29T05:01:46.639 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:46 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-activate[132244]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-29T05:01:46.640 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:46 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-activate[132244]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-29T05:01:46.640 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:46 smithi181 bash[132229]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-29T05:01:46.640 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:46 smithi181 bash[132229]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-29T05:01:46.640 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:46 smithi181 bash[132229]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-29T05:01:46.640 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:46 smithi181 bash[132229]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-29T05:01:46.640 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:46 smithi181 bash[132229]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-03-29T05:01:46.640 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:46 smithi181 bash[132229]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-29T05:01:46.640 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:46 smithi181 bash[132229]: --> ceph-volume raw activate successful for osd ID: 3 2024-03-29T05:01:46.640 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:46 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-activate[132244]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-29T05:01:46.640 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:46 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-activate[132244]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-29T05:01:46.640 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:46 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-activate[132244]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-03-29T05:01:46.640 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:46 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-activate[132244]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-29T05:01:46.640 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:46 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-activate[132244]: --> ceph-volume raw activate successful for osd ID: 3 2024-03-29T05:01:46.640 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:46 smithi181 podman[132229]: 2024-03-29 05:01:46.321357502 +0000 UTC m=+0.807307463 container died c9d1e7c28f086a9bc40d0cf857e7333bbe2a12164f01f6a93c37a66af9227393 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-activate, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-03-29T05:01:46.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:46 smithi181 ceph-mon[106466]: failed to encode map e548 with expected crc 2024-03-29T05:01:46.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:46 smithi181 ceph-mon[106466]: failed to encode map e548 with expected crc 2024-03-29T05:01:46.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:46 smithi181 ceph-mon[106466]: pgmap v490: 105 pgs: 19 stale+active+clean, 86 active+clean; 585 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 16 KiB/s wr, 115 op/s 2024-03-29T05:01:46.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:46 smithi181 ceph-mon[106466]: osdmap e549: 8 total, 7 up, 8 in 2024-03-29T05:01:46.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:46 smithi181 ceph-mon[101970]: failed to encode map e548 with expected crc 2024-03-29T05:01:46.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:46 smithi181 ceph-mon[101970]: failed to encode map e548 with expected crc 2024-03-29T05:01:46.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:46 smithi181 ceph-mon[101970]: pgmap v490: 105 pgs: 19 stale+active+clean, 86 active+clean; 585 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 16 KiB/s wr, 115 op/s 2024-03-29T05:01:46.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:46 smithi181 ceph-mon[101970]: osdmap e549: 8 total, 7 up, 8 in 2024-03-29T05:01:47.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:46 smithi187 ceph-mon[93536]: failed to encode map e548 with expected crc 2024-03-29T05:01:47.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:46 smithi187 ceph-mon[93536]: failed to encode map e548 with expected crc 2024-03-29T05:01:47.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:46 smithi187 ceph-mon[93536]: pgmap v490: 105 pgs: 19 stale+active+clean, 86 active+clean; 585 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 16 KiB/s wr, 115 op/s 2024-03-29T05:01:47.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:46 smithi187 ceph-mon[93536]: osdmap e549: 8 total, 7 up, 8 in 2024-03-29T05:01:47.347 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:46 smithi181 podman[132391]: 2024-03-29 05:01:46.983438677 +0000 UTC m=+0.657169350 container remove c9d1e7c28f086a9bc40d0cf857e7333bbe2a12164f01f6a93c37a66af9227393 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T05:01:47.347 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:47 smithi181 podman[132463]: 2024-03-29 05:01:47.104084604 +0000 UTC m=+0.023561661 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:01:47.347 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:47 smithi181 podman[132463]: 2024-03-29 05:01:47.23306646 +0000 UTC m=+0.152543496 container create 6ac673c0ed7c9b06f751a6b0e5ef3fc822e33217398084d891807d612e7ab377 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, ceph=True) 2024-03-29T05:01:47.547 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.SnapshotCopyupZerosMigration (999 ms) 2024-03-29T05:01:47.548 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.ResizeCopyup 2024-03-29T05:01:47.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:47 smithi181 ceph-mon[101970]: failed to encode map e549 with expected crc 2024-03-29T05:01:47.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:47 smithi181 ceph-mon[101970]: failed to encode map e549 with expected crc 2024-03-29T05:01:47.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:47 smithi181 ceph-mon[101970]: failed to encode map e549 with expected crc 2024-03-29T05:01:47.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:47 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1319157970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:47.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:47 smithi181 ceph-mon[101970]: failed to encode map e550 with expected crc 2024-03-29T05:01:47.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:47 smithi181 ceph-mon[101970]: osdmap e550: 8 total, 7 up, 8 in 2024-03-29T05:01:47.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:47 smithi181 ceph-mon[106466]: failed to encode map e549 with expected crc 2024-03-29T05:01:47.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:47 smithi181 ceph-mon[106466]: failed to encode map e549 with expected crc 2024-03-29T05:01:47.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:47 smithi181 ceph-mon[106466]: failed to encode map e549 with expected crc 2024-03-29T05:01:47.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:47 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1319157970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:47.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:47 smithi181 ceph-mon[106466]: failed to encode map e550 with expected crc 2024-03-29T05:01:47.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:47 smithi181 ceph-mon[106466]: osdmap e550: 8 total, 7 up, 8 in 2024-03-29T05:01:47.956 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:47 smithi181 podman[132463]: 2024-03-29 05:01:47.510128049 +0000 UTC m=+0.429605086 container init 6ac673c0ed7c9b06f751a6b0e5ef3fc822e33217398084d891807d612e7ab377 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, RELEASE=squid-8da2362) 2024-03-29T05:01:47.956 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:47 smithi181 podman[132463]: 2024-03-29 05:01:47.516032431 +0000 UTC m=+0.435509468 container start 6ac673c0ed7c9b06f751a6b0e5ef3fc822e33217398084d891807d612e7ab377 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-03-29T05:01:47.956 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:47 smithi181 ceph-osd[132524]: -- 172.21.15.181:0/562978168 <== mon.2 v2:172.21.15.181:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5597eecfb680 con 0x5597eecc5000 2024-03-29T05:01:47.956 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:47 smithi181 bash[132463]: 6ac673c0ed7c9b06f751a6b0e5ef3fc822e33217398084d891807d612e7ab377 2024-03-29T05:01:47.956 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:47 smithi181 systemd[1]: Started Ceph osd.3 for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T05:01:48.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:47 smithi187 ceph-mon[93536]: failed to encode map e549 with expected crc 2024-03-29T05:01:48.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:47 smithi187 ceph-mon[93536]: failed to encode map e549 with expected crc 2024-03-29T05:01:48.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:47 smithi187 ceph-mon[93536]: failed to encode map e549 with expected crc 2024-03-29T05:01:48.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:47 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1319157970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:48.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:47 smithi187 ceph-mon[93536]: failed to encode map e550 with expected crc 2024-03-29T05:01:48.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:47 smithi187 ceph-mon[93536]: osdmap e550: 8 total, 7 up, 8 in 2024-03-29T05:01:48.454 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3[132498]: 2024-03-29T05:01:48.060+0000 7f17c8f217c0 -1 Falling back to public interface 2024-03-29T05:01:48.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[101970]: failed to encode map e550 with expected crc 2024-03-29T05:01:48.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[101970]: failed to encode map e550 with expected crc 2024-03-29T05:01:48.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[101970]: failed to encode map e550 with expected crc 2024-03-29T05:01:48.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[101970]: pgmap v493: 105 pgs: 16 active+undersized+degraded, 28 active+undersized, 61 active+clean; 587 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 12 KiB/s wr, 276 op/s; 36/348 objects degraded (10.345%) 2024-03-29T05:01:48.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:48.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:48.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:01:48.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[101970]: Health check failed: Degraded data redundancy: 36/348 objects degraded (10.345%), 16 pgs degraded (PG_DEGRADED) 2024-03-29T05:01:48.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[101970]: failed to encode map e551 with expected crc 2024-03-29T05:01:48.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[101970]: osdmap e551: 8 total, 7 up, 8 in 2024-03-29T05:01:48.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[106466]: failed to encode map e550 with expected crc 2024-03-29T05:01:48.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[106466]: failed to encode map e550 with expected crc 2024-03-29T05:01:48.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[106466]: failed to encode map e550 with expected crc 2024-03-29T05:01:48.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[106466]: pgmap v493: 105 pgs: 16 active+undersized+degraded, 28 active+undersized, 61 active+clean; 587 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 12 KiB/s wr, 276 op/s; 36/348 objects degraded (10.345%) 2024-03-29T05:01:48.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:48.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:48.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:01:48.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[106466]: Health check failed: Degraded data redundancy: 36/348 objects degraded (10.345%), 16 pgs degraded (PG_DEGRADED) 2024-03-29T05:01:48.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[106466]: failed to encode map e551 with expected crc 2024-03-29T05:01:48.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-mon[106466]: osdmap e551: 8 total, 7 up, 8 in 2024-03-29T05:01:49.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:48 smithi187 ceph-mon[93536]: failed to encode map e550 with expected crc 2024-03-29T05:01:49.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:48 smithi187 ceph-mon[93536]: failed to encode map e550 with expected crc 2024-03-29T05:01:49.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:48 smithi187 ceph-mon[93536]: failed to encode map e550 with expected crc 2024-03-29T05:01:49.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:48 smithi187 ceph-mon[93536]: pgmap v493: 105 pgs: 16 active+undersized+degraded, 28 active+undersized, 61 active+clean; 587 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 12 KiB/s wr, 276 op/s; 36/348 objects degraded (10.345%) 2024-03-29T05:01:49.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:48 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:49.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:48 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:49.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:48 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:01:49.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:48 smithi187 ceph-mon[93536]: Health check failed: Degraded data redundancy: 36/348 objects degraded (10.345%), 16 pgs degraded (PG_DEGRADED) 2024-03-29T05:01:49.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:48 smithi187 ceph-mon[93536]: failed to encode map e551 with expected crc 2024-03-29T05:01:49.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:48 smithi187 ceph-mon[93536]: osdmap e551: 8 total, 7 up, 8 in 2024-03-29T05:01:49.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:01:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:01:48] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2024-03-29T05:01:49.572 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.ResizeCopyup (2024 ms) 2024-03-29T05:01:49.572 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.DiscardCopyup 2024-03-29T05:01:49.581 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T05:01:49.582 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.DiscardCopyup (10 ms) 2024-03-29T05:01:49.582 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.ImageOptions 2024-03-29T05:01:49.589 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.ImageOptions (8 ms) 2024-03-29T05:01:49.589 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.WriteFullCopyup 2024-03-29T05:01:49.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:49 smithi187 ceph-mon[93536]: failed to encode map e551 with expected crc 2024-03-29T05:01:49.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:49 smithi187 ceph-mon[93536]: failed to encode map e551 with expected crc 2024-03-29T05:01:49.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:49 smithi187 ceph-mon[93536]: failed to encode map e552 with expected crc 2024-03-29T05:01:49.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:49 smithi187 ceph-mon[93536]: osdmap e552: 8 total, 7 up, 8 in 2024-03-29T05:01:49.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:49 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1127498127' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:49.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:49 smithi181 ceph-mon[101970]: failed to encode map e551 with expected crc 2024-03-29T05:01:49.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:49 smithi181 ceph-mon[101970]: failed to encode map e551 with expected crc 2024-03-29T05:01:49.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:49 smithi181 ceph-mon[101970]: failed to encode map e552 with expected crc 2024-03-29T05:01:49.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:49 smithi181 ceph-mon[101970]: osdmap e552: 8 total, 7 up, 8 in 2024-03-29T05:01:49.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:49 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1127498127' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:49.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:49 smithi181 ceph-mon[106466]: failed to encode map e551 with expected crc 2024-03-29T05:01:49.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:49 smithi181 ceph-mon[106466]: failed to encode map e551 with expected crc 2024-03-29T05:01:49.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:49 smithi181 ceph-mon[106466]: failed to encode map e552 with expected crc 2024-03-29T05:01:49.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:49 smithi181 ceph-mon[106466]: osdmap e552: 8 total, 7 up, 8 in 2024-03-29T05:01:49.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:49 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1127498127' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:51.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:50 smithi187 ceph-mon[93536]: failed to encode map e552 with expected crc 2024-03-29T05:01:51.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:50 smithi187 ceph-mon[93536]: failed to encode map e552 with expected crc 2024-03-29T05:01:51.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:50 smithi187 ceph-mon[93536]: pgmap v496: 105 pgs: 16 active+undersized+degraded, 28 active+undersized, 61 active+clean; 587 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 12 KiB/s wr, 276 op/s; 36/348 objects degraded (10.345%) 2024-03-29T05:01:51.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:50 smithi187 ceph-mon[93536]: failed to encode map e553 with expected crc 2024-03-29T05:01:51.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:50 smithi187 ceph-mon[93536]: osdmap e553: 8 total, 7 up, 8 in 2024-03-29T05:01:51.205 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:50 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3[132498]: 2024-03-29T05:01:50.904+0000 7f17c8f217c0 -1 osd.3 544 log_to_monitors true 2024-03-29T05:01:51.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:50 smithi181 ceph-mon[106466]: failed to encode map e552 with expected crc 2024-03-29T05:01:51.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:50 smithi181 ceph-mon[106466]: failed to encode map e552 with expected crc 2024-03-29T05:01:51.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:50 smithi181 ceph-mon[106466]: pgmap v496: 105 pgs: 16 active+undersized+degraded, 28 active+undersized, 61 active+clean; 587 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 12 KiB/s wr, 276 op/s; 36/348 objects degraded (10.345%) 2024-03-29T05:01:51.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:50 smithi181 ceph-mon[106466]: failed to encode map e553 with expected crc 2024-03-29T05:01:51.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:50 smithi181 ceph-mon[106466]: osdmap e553: 8 total, 7 up, 8 in 2024-03-29T05:01:51.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:50 smithi181 ceph-mon[101970]: failed to encode map e552 with expected crc 2024-03-29T05:01:51.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:50 smithi181 ceph-mon[101970]: failed to encode map e552 with expected crc 2024-03-29T05:01:51.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:50 smithi181 ceph-mon[101970]: pgmap v496: 105 pgs: 16 active+undersized+degraded, 28 active+undersized, 61 active+clean; 587 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 12 KiB/s wr, 276 op/s; 36/348 objects degraded (10.345%) 2024-03-29T05:01:51.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:50 smithi181 ceph-mon[101970]: failed to encode map e553 with expected crc 2024-03-29T05:01:51.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:50 smithi181 ceph-mon[101970]: osdmap e553: 8 total, 7 up, 8 in 2024-03-29T05:01:52.204 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3[132498]: 2024-03-29T05:01:52.033+0000 7f17b6371700 -1 osd.3 544 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T05:01:52.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[106466]: failed to encode map e553 with expected crc 2024-03-29T05:01:52.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[106466]: failed to encode map e553 with expected crc 2024-03-29T05:01:52.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[106466]: failed to encode map e553 with expected crc 2024-03-29T05:01:52.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[106466]: from='osd.3 [v2:172.21.15.181:6826/949048273,v1:172.21.15.181:6827/949048273]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T05:01:52.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[106466]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T05:01:52.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[106466]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T05:01:52.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[106466]: failed to encode map e554 with expected crc 2024-03-29T05:01:52.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[106466]: osdmap e554: 8 total, 7 up, 8 in 2024-03-29T05:01:52.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[106466]: from='osd.3 [v2:172.21.15.181:6826/949048273,v1:172.21.15.181:6827/949048273]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T05:01:52.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[106466]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T05:01:52.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[101970]: failed to encode map e553 with expected crc 2024-03-29T05:01:52.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[101970]: failed to encode map e553 with expected crc 2024-03-29T05:01:52.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[101970]: failed to encode map e553 with expected crc 2024-03-29T05:01:52.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[101970]: from='osd.3 [v2:172.21.15.181:6826/949048273,v1:172.21.15.181:6827/949048273]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T05:01:52.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[101970]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T05:01:52.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[101970]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T05:01:52.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[101970]: failed to encode map e554 with expected crc 2024-03-29T05:01:52.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[101970]: osdmap e554: 8 total, 7 up, 8 in 2024-03-29T05:01:52.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[101970]: from='osd.3 [v2:172.21.15.181:6826/949048273,v1:172.21.15.181:6827/949048273]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T05:01:52.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:51 smithi181 ceph-mon[101970]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T05:01:52.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:51 smithi187 ceph-mon[93536]: failed to encode map e553 with expected crc 2024-03-29T05:01:52.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:51 smithi187 ceph-mon[93536]: failed to encode map e553 with expected crc 2024-03-29T05:01:52.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:51 smithi187 ceph-mon[93536]: failed to encode map e553 with expected crc 2024-03-29T05:01:52.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:51 smithi187 ceph-mon[93536]: from='osd.3 [v2:172.21.15.181:6826/949048273,v1:172.21.15.181:6827/949048273]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T05:01:52.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:51 smithi187 ceph-mon[93536]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T05:01:52.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:51 smithi187 ceph-mon[93536]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T05:01:52.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:51 smithi187 ceph-mon[93536]: failed to encode map e554 with expected crc 2024-03-29T05:01:52.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:51 smithi187 ceph-mon[93536]: osdmap e554: 8 total, 7 up, 8 in 2024-03-29T05:01:52.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:51 smithi187 ceph-mon[93536]: from='osd.3 [v2:172.21.15.181:6826/949048273,v1:172.21.15.181:6827/949048273]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T05:01:52.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:51 smithi187 ceph-mon[93536]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi181", "root=default"]}]: dispatch 2024-03-29T05:01:53.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[106466]: failed to encode map e554 with expected crc 2024-03-29T05:01:53.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[106466]: pgmap v499: 105 pgs: 16 active+undersized+degraded, 28 active+undersized, 61 active+clean; 4.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 111 KiB/s rd, 1.0 MiB/s wr, 146 op/s; 41/390 objects degraded (10.513%) 2024-03-29T05:01:53.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:53.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:53.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:53.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:53.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[106466]: from='osd.3 ' entity='osd.3' 2024-03-29T05:01:53.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1858517307' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:53.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[106466]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T05:01:53.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[106466]: osd.3 [v2:172.21.15.181:6826/949048273,v1:172.21.15.181:6827/949048273] boot 2024-03-29T05:01:53.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[106466]: osdmap e555: 8 total, 8 up, 8 in 2024-03-29T05:01:53.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T05:01:53.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[101970]: failed to encode map e554 with expected crc 2024-03-29T05:01:53.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[101970]: pgmap v499: 105 pgs: 16 active+undersized+degraded, 28 active+undersized, 61 active+clean; 4.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 111 KiB/s rd, 1.0 MiB/s wr, 146 op/s; 41/390 objects degraded (10.513%) 2024-03-29T05:01:53.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:53.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:53.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:53.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:53.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[101970]: from='osd.3 ' entity='osd.3' 2024-03-29T05:01:53.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1858517307' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:53.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[101970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T05:01:53.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[101970]: osd.3 [v2:172.21.15.181:6826/949048273,v1:172.21.15.181:6827/949048273] boot 2024-03-29T05:01:53.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[101970]: osdmap e555: 8 total, 8 up, 8 in 2024-03-29T05:01:53.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:52 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T05:01:53.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:52 smithi187 ceph-mon[93536]: failed to encode map e554 with expected crc 2024-03-29T05:01:53.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:52 smithi187 ceph-mon[93536]: pgmap v499: 105 pgs: 16 active+undersized+degraded, 28 active+undersized, 61 active+clean; 4.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 111 KiB/s rd, 1.0 MiB/s wr, 146 op/s; 41/390 objects degraded (10.513%) 2024-03-29T05:01:53.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:52 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:53.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:52 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:53.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:52 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:53.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:52 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:53.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:52 smithi187 ceph-mon[93536]: from='osd.3 ' entity='osd.3' 2024-03-29T05:01:53.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:52 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1858517307' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:53.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:52 smithi187 ceph-mon[93536]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T05:01:53.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:52 smithi187 ceph-mon[93536]: osd.3 [v2:172.21.15.181:6826/949048273,v1:172.21.15.181:6827/949048273] boot 2024-03-29T05:01:53.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:52 smithi187 ceph-mon[93536]: osdmap e555: 8 total, 8 up, 8 in 2024-03-29T05:01:53.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:52 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T05:01:53.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:53 smithi181 ceph-mon[106466]: failed to encode map e555 with expected crc 2024-03-29T05:01:53.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:53 smithi181 ceph-mon[106466]: failed to encode map e555 with expected crc 2024-03-29T05:01:53.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:53 smithi181 ceph-mon[106466]: failed to encode map e555 with expected crc 2024-03-29T05:01:53.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:53 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:53.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:53 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:53.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:53 smithi181 ceph-mon[106466]: osdmap e556: 8 total, 8 up, 8 in 2024-03-29T05:01:53.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:53 smithi181 ceph-mon[101970]: failed to encode map e555 with expected crc 2024-03-29T05:01:53.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:53 smithi181 ceph-mon[101970]: failed to encode map e555 with expected crc 2024-03-29T05:01:53.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:53 smithi181 ceph-mon[101970]: failed to encode map e555 with expected crc 2024-03-29T05:01:53.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:53 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:53.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:53 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:53.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:53 smithi181 ceph-mon[101970]: osdmap e556: 8 total, 8 up, 8 in 2024-03-29T05:01:54.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:53 smithi187 ceph-mon[93536]: failed to encode map e555 with expected crc 2024-03-29T05:01:54.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:53 smithi187 ceph-mon[93536]: failed to encode map e555 with expected crc 2024-03-29T05:01:54.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:53 smithi187 ceph-mon[93536]: failed to encode map e555 with expected crc 2024-03-29T05:01:54.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:53 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:54.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:53 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:54.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:53 smithi187 ceph-mon[93536]: osdmap e556: 8 total, 8 up, 8 in 2024-03-29T05:01:54.442 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 4 2024-03-29T05:01:54.937 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.WriteFullCopyup (5347 ms) 2024-03-29T05:01:54.937 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.DiffIterateCloneOverwrite 2024-03-29T05:01:55.008 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[106466]: pgmap v501: 105 pgs: 16 active+undersized+degraded, 28 active+undersized, 61 active+clean; 13 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.0 MiB/s wr, 268 op/s; 56/471 objects degraded (11.890%) 2024-03-29T05:01:55.008 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[106466]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.008 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[106466]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.008 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[106466]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.008 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[106466]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.008 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[106466]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.008 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[106466]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.008 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[106466]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.009 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[106466]: Health check update: Degraded data redundancy: 56/471 objects degraded (11.890%), 16 pgs degraded (PG_DEGRADED) 2024-03-29T05:01:55.009 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/907224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:55.009 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[101970]: pgmap v501: 105 pgs: 16 active+undersized+degraded, 28 active+undersized, 61 active+clean; 13 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.0 MiB/s wr, 268 op/s; 56/471 objects degraded (11.890%) 2024-03-29T05:01:55.009 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[101970]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.009 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[101970]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.009 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[101970]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.009 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[101970]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.009 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[101970]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.009 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[101970]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.009 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[101970]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.009 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[101970]: Health check update: Degraded data redundancy: 56/471 objects degraded (11.890%), 16 pgs degraded (PG_DEGRADED) 2024-03-29T05:01:55.009 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:54 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/907224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:55.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:54 smithi187 ceph-mon[93536]: pgmap v501: 105 pgs: 16 active+undersized+degraded, 28 active+undersized, 61 active+clean; 13 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.0 MiB/s wr, 268 op/s; 56/471 objects degraded (11.890%) 2024-03-29T05:01:55.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:54 smithi187 ceph-mon[93536]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:54 smithi187 ceph-mon[93536]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:54 smithi187 ceph-mon[93536]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:54 smithi187 ceph-mon[93536]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:54 smithi187 ceph-mon[93536]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:54 smithi187 ceph-mon[93536]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:54 smithi187 ceph-mon[93536]: failed to encode map e556 with expected crc 2024-03-29T05:01:55.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:54 smithi187 ceph-mon[93536]: Health check update: Degraded data redundancy: 56/471 objects degraded (11.890%), 16 pgs degraded (PG_DEGRADED) 2024-03-29T05:01:55.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:54 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/907224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T05:01:55.516 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade pause' 2024-03-29T05:01:56.043 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:01:56.166 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:55 smithi181 ceph-mon[106466]: failed to encode map e557 with expected crc 2024-03-29T05:01:56.166 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:55 smithi181 ceph-mon[106466]: osdmap e557: 8 total, 8 up, 8 in 2024-03-29T05:01:56.166 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:55 smithi181 ceph-mon[106466]: failed to encode map e557 with expected crc 2024-03-29T05:01:56.166 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:55 smithi181 ceph-mon[106466]: failed to encode map e557 with expected crc 2024-03-29T05:01:56.166 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:55 smithi181 ceph-mon[106466]: failed to encode map e557 with expected crc 2024-03-29T05:01:56.167 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:55 smithi181 ceph-mon[101970]: failed to encode map e557 with expected crc 2024-03-29T05:01:56.167 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:55 smithi181 ceph-mon[101970]: osdmap e557: 8 total, 8 up, 8 in 2024-03-29T05:01:56.167 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:55 smithi181 ceph-mon[101970]: failed to encode map e557 with expected crc 2024-03-29T05:01:56.167 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:55 smithi181 ceph-mon[101970]: failed to encode map e557 with expected crc 2024-03-29T05:01:56.167 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:55 smithi181 ceph-mon[101970]: failed to encode map e557 with expected crc 2024-03-29T05:01:56.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:55 smithi187 ceph-mon[93536]: failed to encode map e557 with expected crc 2024-03-29T05:01:56.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:55 smithi187 ceph-mon[93536]: osdmap e557: 8 total, 8 up, 8 in 2024-03-29T05:01:56.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:55 smithi187 ceph-mon[93536]: failed to encode map e557 with expected crc 2024-03-29T05:01:56.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:55 smithi187 ceph-mon[93536]: failed to encode map e557 with expected crc 2024-03-29T05:01:56.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:55 smithi187 ceph-mon[93536]: failed to encode map e557 with expected crc 2024-03-29T05:01:56.955 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.DiffIterateCloneOverwrite (2018 ms) 2024-03-29T05:01:56.955 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.TestCoR 2024-03-29T05:01:56.964 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING due to disabled rbd_copy_on_read 2024-03-29T05:01:56.964 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.TestCoR (9 ms) 2024-03-29T05:01:56.964 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.FlattenNoEmptyObjects 2024-03-29T05:01:57.163 INFO:tasks.workunit.client.0.smithi181.stdout:made parent image "image397": 4194304K (1024 * 4K) 2024-03-29T05:01:57.163 INFO:tasks.workunit.client.0.smithi181.stdout:generated random write map: 2024-03-29T05:01:57.163 INFO:tasks.workunit.client.0.smithi181.stdout: [11 , 47355 ] 2024-03-29T05:01:57.163 INFO:tasks.workunit.client.0.smithi181.stdout: [15 , 62504 ] 2024-03-29T05:01:57.163 INFO:tasks.workunit.client.0.smithi181.stdout: [20 , 82578 ] 2024-03-29T05:01:57.163 INFO:tasks.workunit.client.0.smithi181.stdout: [25 , 103895 ] 2024-03-29T05:01:57.163 INFO:tasks.workunit.client.0.smithi181.stdout: [32 , 131636 ] 2024-03-29T05:01:57.163 INFO:tasks.workunit.client.0.smithi181.stdout: [54 , 223975 ] 2024-03-29T05:01:57.163 INFO:tasks.workunit.client.0.smithi181.stdout: [71 , 291863 ] 2024-03-29T05:01:57.163 INFO:tasks.workunit.client.0.smithi181.stdout: [88 , 362875 ] 2024-03-29T05:01:57.163 INFO:tasks.workunit.client.0.smithi181.stdout: [93 , 382243 ] 2024-03-29T05:01:57.163 INFO:tasks.workunit.client.0.smithi181.stdout: [95 , 389585 ] 2024-03-29T05:01:57.163 INFO:tasks.workunit.client.0.smithi181.stdout: [97 , 397831 ] 2024-03-29T05:01:57.163 INFO:tasks.workunit.client.0.smithi181.stdout: [106 , 435021 ] 2024-03-29T05:01:57.164 INFO:tasks.workunit.client.0.smithi181.stdout: [111 , 456868 ] 2024-03-29T05:01:57.164 INFO:tasks.workunit.client.0.smithi181.stdout: [122 , 499999 ] 2024-03-29T05:01:57.164 INFO:tasks.workunit.client.0.smithi181.stdout: [145 , 594654 ] 2024-03-29T05:01:57.164 INFO:tasks.workunit.client.0.smithi181.stdout: [146 , 598306 ] 2024-03-29T05:01:57.164 INFO:tasks.workunit.client.0.smithi181.stdout: [148 , 607484 ] 2024-03-29T05:01:57.164 INFO:tasks.workunit.client.0.smithi181.stdout: [158 , 650405 ] 2024-03-29T05:01:57.164 INFO:tasks.workunit.client.0.smithi181.stdout: [177 , 726650 ] 2024-03-29T05:01:57.164 INFO:tasks.workunit.client.0.smithi181.stdout: [194 , 795894 ] 2024-03-29T05:01:57.164 INFO:tasks.workunit.client.0.smithi181.stdout: [195 , 799662 ] 2024-03-29T05:01:57.164 INFO:tasks.workunit.client.0.smithi181.stdout: [227 , 931885 ] 2024-03-29T05:01:57.164 INFO:tasks.workunit.client.0.smithi181.stdout: [228 , 935764 ] 2024-03-29T05:01:57.164 INFO:tasks.workunit.client.0.smithi181.stdout: [234 , 960152 ] 2024-03-29T05:01:57.164 INFO:tasks.workunit.client.0.smithi181.stdout: [237 , 973203 ] 2024-03-29T05:01:57.164 INFO:tasks.workunit.client.0.smithi181.stdout: [238 , 975998 ] 2024-03-29T05:01:57.164 INFO:tasks.workunit.client.0.smithi181.stdout: [245 , 1006071 ] 2024-03-29T05:01:57.164 INFO:tasks.workunit.client.0.smithi181.stdout: [255 , 1045219 ] 2024-03-29T05:01:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: [284 , 1166216 ] 2024-03-29T05:01:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: [291 , 1192553 ] 2024-03-29T05:01:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: [313 , 1282360 ] 2024-03-29T05:01:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: [319 , 1308637 ] 2024-03-29T05:01:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: [324 , 1327687 ] 2024-03-29T05:01:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: [325 , 1333303 ] 2024-03-29T05:01:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: [332 , 1361402 ] 2024-03-29T05:01:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: [350 , 1433990 ] 2024-03-29T05:01:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: [353 , 1449353 ] 2024-03-29T05:01:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: [361 , 1479960 ] 2024-03-29T05:01:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: [365 , 1498246 ] 2024-03-29T05:01:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: [385 , 1580297 ] 2024-03-29T05:01:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: [388 , 1591111 ] 2024-03-29T05:01:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: [396 , 1623954 ] 2024-03-29T05:01:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: [401 , 1644971 ] 2024-03-29T05:01:57.166 INFO:tasks.workunit.client.0.smithi181.stdout: [415 , 1702757 ] 2024-03-29T05:01:57.166 INFO:tasks.workunit.client.0.smithi181.stdout: [422 , 1731267 ] 2024-03-29T05:01:57.166 INFO:tasks.workunit.client.0.smithi181.stdout: [444 , 1820386 ] 2024-03-29T05:01:57.166 INFO:tasks.workunit.client.0.smithi181.stdout: [455 , 1863900 ] 2024-03-29T05:01:57.166 INFO:tasks.workunit.client.0.smithi181.stdout: [479 , 1962688 ] 2024-03-29T05:01:57.166 INFO:tasks.workunit.client.0.smithi181.stdout: [480 , 1968066 ] 2024-03-29T05:01:57.166 INFO:tasks.workunit.client.0.smithi181.stdout: [483 , 1978752 ] 2024-03-29T05:01:57.166 INFO:tasks.workunit.client.0.smithi181.stdout: [487 , 1997915 ] 2024-03-29T05:01:57.166 INFO:tasks.workunit.client.0.smithi181.stdout: [498 , 2043176 ] 2024-03-29T05:01:57.166 INFO:tasks.workunit.client.0.smithi181.stdout: [513 , 2103356 ] 2024-03-29T05:01:57.166 INFO:tasks.workunit.client.0.smithi181.stdout: [532 , 2181890 ] 2024-03-29T05:01:57.167 INFO:tasks.workunit.client.0.smithi181.stdout: [535 , 2193593 ] 2024-03-29T05:01:57.167 INFO:tasks.workunit.client.0.smithi181.stdout: [539 , 2210647 ] 2024-03-29T05:01:57.167 INFO:tasks.workunit.client.0.smithi181.stdout: [543 , 2225675 ] 2024-03-29T05:01:57.167 INFO:tasks.workunit.client.0.smithi181.stdout: [583 , 2390012 ] 2024-03-29T05:01:57.167 INFO:tasks.workunit.client.0.smithi181.stdout: [589 , 2414776 ] 2024-03-29T05:01:57.167 INFO:tasks.workunit.client.0.smithi181.stdout: [603 , 2470749 ] 2024-03-29T05:01:57.167 INFO:tasks.workunit.client.0.smithi181.stdout: [617 , 2530102 ] 2024-03-29T05:01:57.167 INFO:tasks.workunit.client.0.smithi181.stdout: [626 , 2565139 ] 2024-03-29T05:01:57.167 INFO:tasks.workunit.client.0.smithi181.stdout: [635 , 2602068 ] 2024-03-29T05:01:57.168 INFO:tasks.workunit.client.0.smithi181.stdout: [644 , 2639947 ] 2024-03-29T05:01:57.168 INFO:tasks.workunit.client.0.smithi181.stdout: [668 , 2737082 ] 2024-03-29T05:01:57.168 INFO:tasks.workunit.client.0.smithi181.stdout: [669 , 2741959 ] 2024-03-29T05:01:57.168 INFO:tasks.workunit.client.0.smithi181.stdout: [672 , 2754009 ] 2024-03-29T05:01:57.168 INFO:tasks.workunit.client.0.smithi181.stdout: [675 , 2766531 ] 2024-03-29T05:01:57.168 INFO:tasks.workunit.client.0.smithi181.stdout: [683 , 2799519 ] 2024-03-29T05:01:57.169 INFO:tasks.workunit.client.0.smithi181.stdout: [697 , 2856178 ] 2024-03-29T05:01:57.169 INFO:tasks.workunit.client.0.smithi181.stdout: [708 , 2901820 ] 2024-03-29T05:01:57.169 INFO:tasks.workunit.client.0.smithi181.stdout: [710 , 2908267 ] 2024-03-29T05:01:57.169 INFO:tasks.workunit.client.0.smithi181.stdout: [715 , 2930603 ] 2024-03-29T05:01:57.169 INFO:tasks.workunit.client.0.smithi181.stdout: [719 , 2947748 ] 2024-03-29T05:01:57.169 INFO:tasks.workunit.client.0.smithi181.stdout: [740 , 3033596 ] 2024-03-29T05:01:57.169 INFO:tasks.workunit.client.0.smithi181.stdout: [744 , 3049241 ] 2024-03-29T05:01:57.169 INFO:tasks.workunit.client.0.smithi181.stdout: [746 , 3057803 ] 2024-03-29T05:01:57.169 INFO:tasks.workunit.client.0.smithi181.stdout: [750 , 3075290 ] 2024-03-29T05:01:57.169 INFO:tasks.workunit.client.0.smithi181.stdout: [762 , 3124656 ] 2024-03-29T05:01:57.169 INFO:tasks.workunit.client.0.smithi181.stdout: [774 , 3173045 ] 2024-03-29T05:01:57.169 INFO:tasks.workunit.client.0.smithi181.stdout: [816 , 3345634 ] 2024-03-29T05:01:57.169 INFO:tasks.workunit.client.0.smithi181.stdout: [820 , 3359691 ] 2024-03-29T05:01:57.169 INFO:tasks.workunit.client.0.smithi181.stdout: [845 , 3463037 ] 2024-03-29T05:01:57.170 INFO:tasks.workunit.client.0.smithi181.stdout: [851 , 3486662 ] 2024-03-29T05:01:57.170 INFO:tasks.workunit.client.0.smithi181.stdout: [853 , 3495140 ] 2024-03-29T05:01:57.170 INFO:tasks.workunit.client.0.smithi181.stdout: [866 , 3550361 ] 2024-03-29T05:01:57.170 INFO:tasks.workunit.client.0.smithi181.stdout: [880 , 3606434 ] 2024-03-29T05:01:57.170 INFO:tasks.workunit.client.0.smithi181.stdout: [901 , 3692622 ] 2024-03-29T05:01:57.171 INFO:tasks.workunit.client.0.smithi181.stdout: [934 , 3826067 ] 2024-03-29T05:01:57.171 INFO:tasks.workunit.client.0.smithi181.stdout: [935 , 3831136 ] 2024-03-29T05:01:57.171 INFO:tasks.workunit.client.0.smithi181.stdout: [938 , 3844352 ] 2024-03-29T05:01:57.171 INFO:tasks.workunit.client.0.smithi181.stdout: [950 , 3892803 ] 2024-03-29T05:01:57.171 INFO:tasks.workunit.client.0.smithi181.stdout: [957 , 3922919 ] 2024-03-29T05:01:57.171 INFO:tasks.workunit.client.0.smithi181.stdout: [960 , 3933175 ] 2024-03-29T05:01:57.171 INFO:tasks.workunit.client.0.smithi181.stdout: [976 , 3997917 ] 2024-03-29T05:01:57.171 INFO:tasks.workunit.client.0.smithi181.stdout: [1012 , 4147437 ] 2024-03-29T05:01:57.171 INFO:tasks.workunit.client.0.smithi181.stdout: [1018 , 4172769 ] 2024-03-29T05:01:57.171 INFO:tasks.workunit.client.0.smithi181.stdout: [1020 , 4180974 ] 2024-03-29T05:01:57.171 INFO:tasks.workunit.client.0.smithi181.stdout:write data based on random map 2024-03-29T05:01:57.171 INFO:tasks.workunit.client.0.smithi181.stdout: write object-11 2024-03-29T05:01:57.171 INFO:tasks.workunit.client.0.smithi181.stdout: write object-15 2024-03-29T05:01:57.172 INFO:tasks.workunit.client.0.smithi181.stdout: write object-20 2024-03-29T05:01:57.172 INFO:tasks.workunit.client.0.smithi181.stdout: write object-25 2024-03-29T05:01:57.172 INFO:tasks.workunit.client.0.smithi181.stdout: write object-32 2024-03-29T05:01:57.172 INFO:tasks.workunit.client.0.smithi181.stdout: write object-54 2024-03-29T05:01:57.172 INFO:tasks.workunit.client.0.smithi181.stdout: write object-71 2024-03-29T05:01:57.172 INFO:tasks.workunit.client.0.smithi181.stdout: write object-88 2024-03-29T05:01:57.172 INFO:tasks.workunit.client.0.smithi181.stdout: write object-93 2024-03-29T05:01:57.172 INFO:tasks.workunit.client.0.smithi181.stdout: write object-95 2024-03-29T05:01:57.172 INFO:tasks.workunit.client.0.smithi181.stdout: write object-97 2024-03-29T05:01:57.172 INFO:tasks.workunit.client.0.smithi181.stdout: write object-106 2024-03-29T05:01:57.172 INFO:tasks.workunit.client.0.smithi181.stdout: write object-111 2024-03-29T05:01:57.173 INFO:tasks.workunit.client.0.smithi181.stdout: write object-122 2024-03-29T05:01:57.173 INFO:tasks.workunit.client.0.smithi181.stdout: write object-145 2024-03-29T05:01:57.173 INFO:tasks.workunit.client.0.smithi181.stdout: write object-146 2024-03-29T05:01:57.173 INFO:tasks.workunit.client.0.smithi181.stdout: write object-148 2024-03-29T05:01:57.173 INFO:tasks.workunit.client.0.smithi181.stdout: write object-158 2024-03-29T05:01:57.173 INFO:tasks.workunit.client.0.smithi181.stdout: write object-177 2024-03-29T05:01:57.173 INFO:tasks.workunit.client.0.smithi181.stdout: write object-194 2024-03-29T05:01:57.173 INFO:tasks.workunit.client.0.smithi181.stdout: write object-195 2024-03-29T05:01:57.173 INFO:tasks.workunit.client.0.smithi181.stdout: write object-227 2024-03-29T05:01:57.173 INFO:tasks.workunit.client.0.smithi181.stdout: write object-228 2024-03-29T05:01:57.173 INFO:tasks.workunit.client.0.smithi181.stdout: write object-234 2024-03-29T05:01:57.173 INFO:tasks.workunit.client.0.smithi181.stdout: write object-237 2024-03-29T05:01:57.173 INFO:tasks.workunit.client.0.smithi181.stdout: write object-238 2024-03-29T05:01:57.173 INFO:tasks.workunit.client.0.smithi181.stdout: write object-245 2024-03-29T05:01:57.174 INFO:tasks.workunit.client.0.smithi181.stdout: write object-255 2024-03-29T05:01:57.174 INFO:tasks.workunit.client.0.smithi181.stdout: write object-284 2024-03-29T05:01:57.174 INFO:tasks.workunit.client.0.smithi181.stdout: write object-291 2024-03-29T05:01:57.174 INFO:tasks.workunit.client.0.smithi181.stdout: write object-313 2024-03-29T05:01:57.174 INFO:tasks.workunit.client.0.smithi181.stdout: write object-319 2024-03-29T05:01:57.174 INFO:tasks.workunit.client.0.smithi181.stdout: write object-324 2024-03-29T05:01:57.174 INFO:tasks.workunit.client.0.smithi181.stdout: write object-325 2024-03-29T05:01:57.174 INFO:tasks.workunit.client.0.smithi181.stdout: write object-332 2024-03-29T05:01:57.174 INFO:tasks.workunit.client.0.smithi181.stdout: write object-350 2024-03-29T05:01:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: write object-353 2024-03-29T05:01:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: write object-361 2024-03-29T05:01:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: write object-365 2024-03-29T05:01:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: write object-385 2024-03-29T05:01:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: write object-388 2024-03-29T05:01:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: write object-396 2024-03-29T05:01:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: write object-401 2024-03-29T05:01:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: write object-415 2024-03-29T05:01:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: write object-422 2024-03-29T05:01:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: write object-444 2024-03-29T05:01:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: write object-455 2024-03-29T05:01:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: write object-479 2024-03-29T05:01:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: write object-480 2024-03-29T05:01:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: write object-483 2024-03-29T05:01:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: write object-487 2024-03-29T05:01:57.176 INFO:tasks.workunit.client.0.smithi181.stdout: write object-498 2024-03-29T05:01:57.176 INFO:tasks.workunit.client.0.smithi181.stdout: write object-513 2024-03-29T05:01:57.176 INFO:tasks.workunit.client.0.smithi181.stdout: write object-532 2024-03-29T05:01:57.176 INFO:tasks.workunit.client.0.smithi181.stdout: write object-535 2024-03-29T05:01:57.176 INFO:tasks.workunit.client.0.smithi181.stdout: write object-539 2024-03-29T05:01:57.176 INFO:tasks.workunit.client.0.smithi181.stdout: write object-543 2024-03-29T05:01:57.176 INFO:tasks.workunit.client.0.smithi181.stdout: write object-583 2024-03-29T05:01:57.176 INFO:tasks.workunit.client.0.smithi181.stdout: write object-589 2024-03-29T05:01:57.176 INFO:tasks.workunit.client.0.smithi181.stdout: write object-603 2024-03-29T05:01:57.176 INFO:tasks.workunit.client.0.smithi181.stdout: write object-617 2024-03-29T05:01:57.176 INFO:tasks.workunit.client.0.smithi181.stdout: write object-626 2024-03-29T05:01:57.176 INFO:tasks.workunit.client.0.smithi181.stdout: write object-635 2024-03-29T05:01:57.176 INFO:tasks.workunit.client.0.smithi181.stdout: write object-644 2024-03-29T05:01:57.177 INFO:tasks.workunit.client.0.smithi181.stdout: write object-668 2024-03-29T05:01:57.177 INFO:tasks.workunit.client.0.smithi181.stdout: write object-669 2024-03-29T05:01:57.177 INFO:tasks.workunit.client.0.smithi181.stdout: write object-672 2024-03-29T05:01:57.177 INFO:tasks.workunit.client.0.smithi181.stdout: write object-675 2024-03-29T05:01:57.177 INFO:tasks.workunit.client.0.smithi181.stdout: write object-683 2024-03-29T05:01:57.177 INFO:tasks.workunit.client.0.smithi181.stdout: write object-697 2024-03-29T05:01:57.177 INFO:tasks.workunit.client.0.smithi181.stdout: write object-708 2024-03-29T05:01:57.177 INFO:tasks.workunit.client.0.smithi181.stdout: write object-710 2024-03-29T05:01:57.177 INFO:tasks.workunit.client.0.smithi181.stdout: write object-715 2024-03-29T05:01:57.177 INFO:tasks.workunit.client.0.smithi181.stdout: write object-719 2024-03-29T05:01:57.177 INFO:tasks.workunit.client.0.smithi181.stdout: write object-740 2024-03-29T05:01:57.177 INFO:tasks.workunit.client.0.smithi181.stdout: write object-744 2024-03-29T05:01:57.177 INFO:tasks.workunit.client.0.smithi181.stdout: write object-746 2024-03-29T05:01:57.177 INFO:tasks.workunit.client.0.smithi181.stdout: write object-750 2024-03-29T05:01:57.177 INFO:tasks.workunit.client.0.smithi181.stdout: write object-762 2024-03-29T05:01:57.177 INFO:tasks.workunit.client.0.smithi181.stdout: write object-774 2024-03-29T05:01:57.178 INFO:tasks.workunit.client.0.smithi181.stdout: write object-816 2024-03-29T05:01:57.178 INFO:tasks.workunit.client.0.smithi181.stdout: write object-820 2024-03-29T05:01:57.178 INFO:tasks.workunit.client.0.smithi181.stdout: write object-845 2024-03-29T05:01:57.178 INFO:tasks.workunit.client.0.smithi181.stdout: write object-851 2024-03-29T05:01:57.178 INFO:tasks.workunit.client.0.smithi181.stdout: write object-853 2024-03-29T05:01:57.178 INFO:tasks.workunit.client.0.smithi181.stdout: write object-866 2024-03-29T05:01:57.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:56 smithi181 ceph-mon[106466]: pgmap v504: 105 pgs: 16 active+undersized+degraded, 28 active+undersized, 61 active+clean; 13 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.0 MiB/s wr, 124 op/s; 56/471 objects degraded (11.890%) 2024-03-29T05:01:57.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:56 smithi181 ceph-mon[106466]: failed to encode map e558 with expected crc 2024-03-29T05:01:57.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:56 smithi181 ceph-mon[106466]: osdmap e558: 8 total, 8 up, 8 in 2024-03-29T05:01:57.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:56 smithi181 ceph-mon[106466]: failed to encode map e558 with expected crc 2024-03-29T05:01:57.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:57.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:01:57.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:56 smithi181 ceph-mon[101970]: pgmap v504: 105 pgs: 16 active+undersized+degraded, 28 active+undersized, 61 active+clean; 13 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.0 MiB/s wr, 124 op/s; 56/471 objects degraded (11.890%) 2024-03-29T05:01:57.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:56 smithi181 ceph-mon[101970]: failed to encode map e558 with expected crc 2024-03-29T05:01:57.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:56 smithi181 ceph-mon[101970]: osdmap e558: 8 total, 8 up, 8 in 2024-03-29T05:01:57.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:56 smithi181 ceph-mon[101970]: failed to encode map e558 with expected crc 2024-03-29T05:01:57.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:57.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:01:57.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:56 smithi187 ceph-mon[93536]: pgmap v504: 105 pgs: 16 active+undersized+degraded, 28 active+undersized, 61 active+clean; 13 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.0 MiB/s wr, 124 op/s; 56/471 objects degraded (11.890%) 2024-03-29T05:01:57.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:56 smithi187 ceph-mon[93536]: failed to encode map e558 with expected crc 2024-03-29T05:01:57.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:56 smithi187 ceph-mon[93536]: osdmap e558: 8 total, 8 up, 8 in 2024-03-29T05:01:57.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:56 smithi187 ceph-mon[93536]: failed to encode map e558 with expected crc 2024-03-29T05:01:57.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:57.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:01:58.122 INFO:tasks.workunit.client.0.smithi181.stdout: write object-880 2024-03-29T05:01:58.123 INFO:tasks.workunit.client.0.smithi181.stdout: write object-901 2024-03-29T05:01:58.123 INFO:tasks.workunit.client.0.smithi181.stdout: write object-934 2024-03-29T05:01:58.123 INFO:tasks.workunit.client.0.smithi181.stdout: write object-935 2024-03-29T05:01:58.123 INFO:tasks.workunit.client.0.smithi181.stdout: write object-938 2024-03-29T05:01:58.123 INFO:tasks.workunit.client.0.smithi181.stdout: write object-950 2024-03-29T05:01:58.123 INFO:tasks.workunit.client.0.smithi181.stdout: write object-957 2024-03-29T05:01:58.123 INFO:tasks.workunit.client.0.smithi181.stdout: write object-960 2024-03-29T05:01:58.123 INFO:tasks.workunit.client.0.smithi181.stdout: write object-976 2024-03-29T05:01:58.123 INFO:tasks.workunit.client.0.smithi181.stdout: write object-1012 2024-03-29T05:01:58.123 INFO:tasks.workunit.client.0.smithi181.stdout: write object-1018 2024-03-29T05:01:58.123 INFO:tasks.workunit.client.0.smithi181.stdout: write object-1020 2024-03-29T05:01:58.123 INFO:tasks.workunit.client.0.smithi181.stdout:verify written data by reading 2024-03-29T05:01:58.124 INFO:tasks.workunit.client.0.smithi181.stdout: read object-11 2024-03-29T05:01:58.124 INFO:tasks.workunit.client.0.smithi181.stdout:made snapshot "image397@parent_snap" and protect it 2024-03-29T05:01:58.124 INFO:tasks.workunit.client.0.smithi181.stdout:made and opened clone "image398" 2024-03-29T05:01:58.124 INFO:tasks.workunit.client.0.smithi181.stdout:flattening clone: "image398" 2024-03-29T05:01:58.124 INFO:tasks.workunit.client.0.smithi181.stdout:check whether child image has the same set of objects as parent 2024-03-29T05:01:58.124 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.FlattenNoEmptyObjects (1159 ms) 2024-03-29T05:01:58.124 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.PoolMetadataConfApply 2024-03-29T05:01:58.149 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:57 smithi181 ceph-mon[101970]: failed to encode map e559 with expected crc 2024-03-29T05:01:58.149 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:57 smithi181 ceph-mon[101970]: failed to encode map e559 with expected crc 2024-03-29T05:01:58.149 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:57 smithi181 ceph-mon[101970]: osdmap e559: 8 total, 8 up, 8 in 2024-03-29T05:01:58.150 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:57 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:58.150 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:57 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:58.150 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:57 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:01:58.150 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:57 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:01:58.150 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:57 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:57 smithi181 ceph-mon[106466]: failed to encode map e559 with expected crc 2024-03-29T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:57 smithi181 ceph-mon[106466]: failed to encode map e559 with expected crc 2024-03-29T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:57 smithi181 ceph-mon[106466]: osdmap e559: 8 total, 8 up, 8 in 2024-03-29T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:57 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:58.150 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:57 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:57 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:57 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:01:58.151 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:57 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:58.196 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.PoolMetadataConfApply (74 ms) 2024-03-29T05:01:58.197 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.Sparsify 2024-03-29T05:01:58.220 INFO:tasks.workunit.client.0.smithi181.stdout:sparsify_supported=1 2024-03-29T05:01:58.220 INFO:tasks.workunit.client.0.smithi181.stdout:sparse_read_supported=1 2024-03-29T05:01:58.246 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.Sparsify (50 ms) 2024-03-29T05:01:58.247 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.SparsifyClone 2024-03-29T05:01:58.262 INFO:tasks.workunit.client.0.smithi181.stdout:sparsify_supported=1 2024-03-29T05:01:58.286 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:57 smithi187 ceph-mon[93536]: failed to encode map e559 with expected crc 2024-03-29T05:01:58.286 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:57 smithi187 ceph-mon[93536]: failed to encode map e559 with expected crc 2024-03-29T05:01:58.286 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:57 smithi187 ceph-mon[93536]: osdmap e559: 8 total, 8 up, 8 in 2024-03-29T05:01:58.286 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:57 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:58.286 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:57 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:58.286 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:57 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:01:58.287 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:57 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:01:58.287 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:57 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:58.454 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:01:58] ENGINE Bus STOPPING 2024-03-29T05:01:58.455 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:01:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T05:01:58.455 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:01:58] ENGINE Bus STOPPED 2024-03-29T05:01:58.455 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:01:58] ENGINE Bus STARTING 2024-03-29T05:01:58.455 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:01:58] ENGINE Serving on http://:::9283 2024-03-29T05:01:58.455 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:01:58] ENGINE Bus STARTED 2024-03-29T05:01:58.640 INFO:teuthology.orchestra.run.smithi181.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:01:59.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:58 smithi187 ceph-mon[93536]: pgmap v507: 105 pgs: 105 active+clean; 4.7 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 230 KiB/s rd, 71 KiB/s wr, 356 op/s; 1.0 MiB/s, 27 keys/s, 7 objects/s recovering 2024-03-29T05:01:59.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:58 smithi187 ceph-mon[93536]: pgmap v508: 105 pgs: 105 active+clean; 4.7 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 227 KiB/s rd, 70 KiB/s wr, 350 op/s; 1012 KiB/s, 27 keys/s, 7 objects/s recovering 2024-03-29T05:01:59.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:58 smithi187 ceph-mon[93536]: pgmap v509: 105 pgs: 105 active+clean; 4.7 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 310 KiB/s rd, 96 KiB/s wr, 480 op/s; 1.4 MiB/s, 37 keys/s, 10 objects/s recovering 2024-03-29T05:01:59.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:58 smithi187 ceph-mon[93536]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/471 objects degraded (11.890%), 16 pgs degraded) 2024-03-29T05:01:59.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:58 smithi187 ceph-mon[93536]: failed to encode map e560 with expected crc 2024-03-29T05:01:59.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:58 smithi187 ceph-mon[93536]: osdmap e560: 8 total, 8 up, 8 in 2024-03-29T05:01:59.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:58 smithi187 ceph-mon[93536]: failed to encode map e560 with expected crc 2024-03-29T05:01:59.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:58 smithi187 ceph-mon[93536]: failed to encode map e560 with expected crc 2024-03-29T05:01:59.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:58 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:01:59.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:58 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:59.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:58 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:59.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:58 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:59.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:58 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T05:01:59.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:58 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3239665902' entity='client.admin' 2024-03-29T05:01:59.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:58 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:59.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:01:58] "GET /metrics HTTP/1.1" 200 36326 "" "Prometheus/2.43.0" 2024-03-29T05:01:59.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[106466]: pgmap v507: 105 pgs: 105 active+clean; 4.7 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 230 KiB/s rd, 71 KiB/s wr, 356 op/s; 1.0 MiB/s, 27 keys/s, 7 objects/s recovering 2024-03-29T05:01:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[106466]: pgmap v508: 105 pgs: 105 active+clean; 4.7 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 227 KiB/s rd, 70 KiB/s wr, 350 op/s; 1012 KiB/s, 27 keys/s, 7 objects/s recovering 2024-03-29T05:01:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[106466]: pgmap v509: 105 pgs: 105 active+clean; 4.7 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 310 KiB/s rd, 96 KiB/s wr, 480 op/s; 1.4 MiB/s, 37 keys/s, 10 objects/s recovering 2024-03-29T05:01:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[106466]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/471 objects degraded (11.890%), 16 pgs degraded) 2024-03-29T05:01:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[106466]: failed to encode map e560 with expected crc 2024-03-29T05:01:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[106466]: osdmap e560: 8 total, 8 up, 8 in 2024-03-29T05:01:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[106466]: failed to encode map e560 with expected crc 2024-03-29T05:01:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[106466]: failed to encode map e560 with expected crc 2024-03-29T05:01:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:01:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T05:01:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3239665902' entity='client.admin' 2024-03-29T05:01:59.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:59.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[101970]: pgmap v507: 105 pgs: 105 active+clean; 4.7 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 230 KiB/s rd, 71 KiB/s wr, 356 op/s; 1.0 MiB/s, 27 keys/s, 7 objects/s recovering 2024-03-29T05:01:59.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[101970]: pgmap v508: 105 pgs: 105 active+clean; 4.7 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 227 KiB/s rd, 70 KiB/s wr, 350 op/s; 1012 KiB/s, 27 keys/s, 7 objects/s recovering 2024-03-29T05:01:59.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[101970]: pgmap v509: 105 pgs: 105 active+clean; 4.7 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 310 KiB/s rd, 96 KiB/s wr, 480 op/s; 1.4 MiB/s, 37 keys/s, 10 objects/s recovering 2024-03-29T05:01:59.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[101970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/471 objects degraded (11.890%), 16 pgs degraded) 2024-03-29T05:01:59.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[101970]: failed to encode map e560 with expected crc 2024-03-29T05:01:59.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[101970]: osdmap e560: 8 total, 8 up, 8 in 2024-03-29T05:01:59.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[101970]: failed to encode map e560 with expected crc 2024-03-29T05:01:59.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[101970]: failed to encode map e560 with expected crc 2024-03-29T05:01:59.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:01:59.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:59.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:59.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:01:59.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T05:01:59.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3239665902' entity='client.admin' 2024-03-29T05:01:59.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:58 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:01:59.261 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T05:01:59.518 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:02:00.158 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.SparsifyClone (1912 ms) 2024-03-29T05:02:00.159 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestInternal.MissingDataPool 2024-03-29T05:02:00.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[106466]: failed to encode map e560 with expected crc 2024-03-29T05:02:00.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[106466]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T05:02:00.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[106466]: Upgrade: osd.4 is safe to restart 2024-03-29T05:02:00.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[106466]: from='client.44450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T05:02:00.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[106466]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:02:00.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[106466]: failed to encode map e561 with expected crc 2024-03-29T05:02:00.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[106466]: osdmap e561: 8 total, 8 up, 8 in 2024-03-29T05:02:00.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[106466]: failed to encode map e561 with expected crc 2024-03-29T05:02:00.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[106466]: failed to encode map e561 with expected crc 2024-03-29T05:02:00.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:00.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T05:02:00.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:02:00.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[101970]: failed to encode map e560 with expected crc 2024-03-29T05:02:00.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[101970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T05:02:00.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[101970]: Upgrade: osd.4 is safe to restart 2024-03-29T05:02:00.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[101970]: from='client.44450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T05:02:00.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[101970]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:02:00.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[101970]: failed to encode map e561 with expected crc 2024-03-29T05:02:00.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[101970]: osdmap e561: 8 total, 8 up, 8 in 2024-03-29T05:02:00.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[101970]: failed to encode map e561 with expected crc 2024-03-29T05:02:00.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[101970]: failed to encode map e561 with expected crc 2024-03-29T05:02:00.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:00.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T05:02:00.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:01:59 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:02:00.227 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:59 smithi187 ceph-mon[93536]: failed to encode map e560 with expected crc 2024-03-29T05:02:00.227 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:59 smithi187 ceph-mon[93536]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T05:02:00.227 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:59 smithi187 ceph-mon[93536]: Upgrade: osd.4 is safe to restart 2024-03-29T05:02:00.227 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:59 smithi187 ceph-mon[93536]: from='client.44450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T05:02:00.227 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:59 smithi187 ceph-mon[93536]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:02:00.227 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:59 smithi187 ceph-mon[93536]: failed to encode map e561 with expected crc 2024-03-29T05:02:00.227 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:59 smithi187 ceph-mon[93536]: osdmap e561: 8 total, 8 up, 8 in 2024-03-29T05:02:00.227 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:59 smithi187 ceph-mon[93536]: failed to encode map e561 with expected crc 2024-03-29T05:02:00.227 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:59 smithi187 ceph-mon[93536]: failed to encode map e561 with expected crc 2024-03-29T05:02:00.227 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:59 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:00.227 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:59 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T05:02:00.227 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:01:59 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:02:01.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:00 smithi181 ceph-mon[101970]: Upgrade: Updating osd.4 2024-03-29T05:02:01.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:00 smithi181 ceph-mon[101970]: Deploying daemon osd.4 on smithi187 2024-03-29T05:02:01.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:00 smithi181 ceph-mon[101970]: pgmap v512: 105 pgs: 105 active+clean; 4.7 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 229 KiB/s rd, 91 KiB/s wr, 375 op/s; 145 B/s, 6 keys/s, 3 objects/s recovering 2024-03-29T05:02:01.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:00 smithi181 ceph-mon[101970]: osdmap e562: 8 total, 8 up, 8 in 2024-03-29T05:02:01.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:00 smithi181 ceph-mon[101970]: failed to encode map e562 with expected crc 2024-03-29T05:02:01.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:00 smithi181 ceph-mon[101970]: failed to encode map e562 with expected crc 2024-03-29T05:02:01.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:00 smithi181 ceph-mon[101970]: failed to encode map e562 with expected crc 2024-03-29T05:02:01.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:00 smithi181 ceph-mon[106466]: Upgrade: Updating osd.4 2024-03-29T05:02:01.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:00 smithi181 ceph-mon[106466]: Deploying daemon osd.4 on smithi187 2024-03-29T05:02:01.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:00 smithi181 ceph-mon[106466]: pgmap v512: 105 pgs: 105 active+clean; 4.7 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 229 KiB/s rd, 91 KiB/s wr, 375 op/s; 145 B/s, 6 keys/s, 3 objects/s recovering 2024-03-29T05:02:01.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:00 smithi181 ceph-mon[106466]: osdmap e562: 8 total, 8 up, 8 in 2024-03-29T05:02:01.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:00 smithi181 ceph-mon[106466]: failed to encode map e562 with expected crc 2024-03-29T05:02:01.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:00 smithi181 ceph-mon[106466]: failed to encode map e562 with expected crc 2024-03-29T05:02:01.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:00 smithi181 ceph-mon[106466]: failed to encode map e562 with expected crc 2024-03-29T05:02:01.240 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:00 smithi187 ceph-mon[93536]: Upgrade: Updating osd.4 2024-03-29T05:02:01.240 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:00 smithi187 ceph-mon[93536]: Deploying daemon osd.4 on smithi187 2024-03-29T05:02:01.240 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:00 smithi187 ceph-mon[93536]: pgmap v512: 105 pgs: 105 active+clean; 4.7 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 229 KiB/s rd, 91 KiB/s wr, 375 op/s; 145 B/s, 6 keys/s, 3 objects/s recovering 2024-03-29T05:02:01.241 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:00 smithi187 ceph-mon[93536]: osdmap e562: 8 total, 8 up, 8 in 2024-03-29T05:02:01.241 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:00 smithi187 ceph-mon[93536]: failed to encode map e562 with expected crc 2024-03-29T05:02:01.241 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:00 smithi187 ceph-mon[93536]: failed to encode map e562 with expected crc 2024-03-29T05:02:01.241 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:00 smithi187 ceph-mon[93536]: failed to encode map e562 with expected crc 2024-03-29T05:02:01.253 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestInternal.MissingDataPool (1095 ms) 2024-03-29T05:02:01.400 INFO:teuthology.orchestra.run.smithi181.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T05:02:01.400 INFO:teuthology.orchestra.run.smithi181.stdout:alertmanager.a smithi181 *:9093,9094 running (7m) 9s ago 11m 25.4M - 0.25.0 c8568f914cd2 69e0ad67b57f 2024-03-29T05:02:01.400 INFO:teuthology.orchestra.run.smithi181.stdout:grafana.a smithi187 *:3000 error 9s ago 10m - - 2024-03-29T05:02:01.400 INFO:teuthology.orchestra.run.smithi181.stdout:mds.foo.smithi181.pflrwm smithi181 running (9m) 9s ago 9m 18.3M - 17.2.7-789-g4213e17c cf22eb8683be 0f200e33dce6 2024-03-29T05:02:01.400 INFO:teuthology.orchestra.run.smithi181.stdout:mds.foo.smithi187.vcsnqx smithi187 running (9m) 9s ago 9m 17.4M - 17.2.7-789-g4213e17c cf22eb8683be 245621ed14ed 2024-03-29T05:02:01.400 INFO:teuthology.orchestra.run.smithi181.stdout:mgr.x smithi187 *:8443,9283,8765 running (6m) 9s ago 15m 451M - 19.0.0-1959-g8da23624 54112f57da44 19bc315720bd 2024-03-29T05:02:01.400 INFO:teuthology.orchestra.run.smithi181.stdout:mgr.y smithi181 *:8443,9283,8765 running (6m) 9s ago 16m 509M - 19.0.0-1959-g8da23624 54112f57da44 b07d84f4244a 2024-03-29T05:02:01.400 INFO:teuthology.orchestra.run.smithi181.stdout:mon.a smithi181 running (5m) 9s ago 17m 62.6M 2048M 19.0.0-1959-g8da23624 54112f57da44 4e5d3325d991 2024-03-29T05:02:01.400 INFO:teuthology.orchestra.run.smithi181.stdout:mon.b smithi187 running (4m) 9s ago 15m 48.9M 2048M 19.0.0-1959-g8da23624 54112f57da44 4de0ae3c44c3 2024-03-29T05:02:01.400 INFO:teuthology.orchestra.run.smithi181.stdout:mon.c smithi181 running (4m) 9s ago 15m 55.3M 2048M 19.0.0-1959-g8da23624 54112f57da44 10f02d533230 2024-03-29T05:02:01.400 INFO:teuthology.orchestra.run.smithi181.stdout:node-exporter.a smithi181 *:9100 running (7m) 9s ago 10m 16.5M - 1.5.0 0da6a335fe13 4029d029600b 2024-03-29T05:02:01.400 INFO:teuthology.orchestra.run.smithi181.stdout:node-exporter.b smithi187 *:9100 running (6m) 9s ago 10m 15.5M - 1.5.0 0da6a335fe13 3894f6e97d55 2024-03-29T05:02:01.400 INFO:teuthology.orchestra.run.smithi181.stdout:osd.0 smithi181 running (2m) 9s ago 14m 73.1M 2480M 19.0.0-1959-g8da23624 54112f57da44 03d36b79ef26 2024-03-29T05:02:01.401 INFO:teuthology.orchestra.run.smithi181.stdout:osd.1 smithi181 running (84s) 9s ago 14m 60.6M 2480M 19.0.0-1959-g8da23624 54112f57da44 cb0b10fc92dd 2024-03-29T05:02:01.401 INFO:teuthology.orchestra.run.smithi181.stdout:osd.2 smithi181 running (49s) 9s ago 14m 57.0M 2480M 19.0.0-1959-g8da23624 54112f57da44 12c7ca8de226 2024-03-29T05:02:01.401 INFO:teuthology.orchestra.run.smithi181.stdout:osd.3 smithi181 running (14s) 9s ago 13m 11.8M 2480M 19.0.0-1959-g8da23624 54112f57da44 6ac673c0ed7c 2024-03-29T05:02:01.401 INFO:teuthology.orchestra.run.smithi181.stdout:osd.4 smithi187 running (13m) 9s ago 13m 140M 2480M 17.2.7-789-g4213e17c cf22eb8683be 42698769ef84 2024-03-29T05:02:01.401 INFO:teuthology.orchestra.run.smithi181.stdout:osd.5 smithi187 running (12m) 9s ago 12m 117M 2480M 17.2.7-789-g4213e17c cf22eb8683be 9599ff85a9e7 2024-03-29T05:02:01.401 INFO:teuthology.orchestra.run.smithi181.stdout:osd.6 smithi187 running (12m) 9s ago 12m 143M 2480M 17.2.7-789-g4213e17c cf22eb8683be c3287470c1fa 2024-03-29T05:02:01.401 INFO:teuthology.orchestra.run.smithi181.stdout:osd.7 smithi187 running (12m) 9s ago 12m 120M 2480M 17.2.7-789-g4213e17c cf22eb8683be 89fb8ccf8705 2024-03-29T05:02:01.401 INFO:teuthology.orchestra.run.smithi181.stdout:prometheus.a smithi187 *:9095 running (6m) 9s ago 11m 60.3M - 2.43.0 a07b618ecd1d 882b8922d74b 2024-03-29T05:02:01.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:01 smithi181 ceph-mon[106466]: failed to encode map e562 with expected crc 2024-03-29T05:02:01.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:01 smithi181 ceph-mon[106466]: osdmap e563: 8 total, 8 up, 8 in 2024-03-29T05:02:01.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:01 smithi181 ceph-mon[106466]: failed to encode map e563 with expected crc 2024-03-29T05:02:01.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:01 smithi181 ceph-mon[106466]: failed to encode map e563 with expected crc 2024-03-29T05:02:01.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:01 smithi181 ceph-mon[106466]: failed to encode map e563 with expected crc 2024-03-29T05:02:01.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:01 smithi181 ceph-mon[101970]: failed to encode map e562 with expected crc 2024-03-29T05:02:01.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:01 smithi181 ceph-mon[101970]: osdmap e563: 8 total, 8 up, 8 in 2024-03-29T05:02:01.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:01 smithi181 ceph-mon[101970]: failed to encode map e563 with expected crc 2024-03-29T05:02:01.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:01 smithi181 ceph-mon[101970]: failed to encode map e563 with expected crc 2024-03-29T05:02:01.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:01 smithi181 ceph-mon[101970]: failed to encode map e563 with expected crc 2024-03-29T05:02:02.005 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T05:02:02.159 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 34 tests from TestInternal (41740 ms total) 2024-03-29T05:02:02.159 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:02:02.159 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 17 tests from TestImageWatcher 2024-03-29T05:02:02.266 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:02:02.314 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:01 smithi187 ceph-mon[93536]: failed to encode map e562 with expected crc 2024-03-29T05:02:02.314 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:01 smithi187 ceph-mon[93536]: osdmap e563: 8 total, 8 up, 8 in 2024-03-29T05:02:02.314 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:01 smithi187 ceph-mon[93536]: failed to encode map e563 with expected crc 2024-03-29T05:02:02.314 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:01 smithi187 ceph-mon[93536]: failed to encode map e563 with expected crc 2024-03-29T05:02:02.314 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:01 smithi187 ceph-mon[93536]: failed to encode map e563 with expected crc 2024-03-29T05:02:03.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:02 smithi181 ceph-mon[106466]: failed to encode map e563 with expected crc 2024-03-29T05:02:03.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:02 smithi181 ceph-mon[106466]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T05:02:03.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:02 smithi181 ceph-mon[106466]: pgmap v515: 105 pgs: 105 active+clean; 4.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 324 KiB/s rd, 101 KiB/s wr, 486 op/s 2024-03-29T05:02:03.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:02 smithi181 ceph-mon[106466]: failed to encode map e564 with expected crc 2024-03-29T05:02:03.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:02 smithi181 ceph-mon[106466]: osdmap e564: 8 total, 8 up, 8 in 2024-03-29T05:02:03.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:02 smithi181 ceph-mon[106466]: failed to encode map e564 with expected crc 2024-03-29T05:02:03.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:02 smithi181 ceph-mon[101970]: failed to encode map e563 with expected crc 2024-03-29T05:02:03.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:02 smithi181 ceph-mon[101970]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T05:02:03.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:02 smithi181 ceph-mon[101970]: pgmap v515: 105 pgs: 105 active+clean; 4.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 324 KiB/s rd, 101 KiB/s wr, 486 op/s 2024-03-29T05:02:03.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:02 smithi181 ceph-mon[101970]: failed to encode map e564 with expected crc 2024-03-29T05:02:03.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:02 smithi181 ceph-mon[101970]: osdmap e564: 8 total, 8 up, 8 in 2024-03-29T05:02:03.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:02 smithi181 ceph-mon[101970]: failed to encode map e564 with expected crc 2024-03-29T05:02:03.214 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:02 smithi187 ceph-mon[93536]: failed to encode map e563 with expected crc 2024-03-29T05:02:03.214 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:02 smithi187 ceph-mon[93536]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T05:02:03.215 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:02 smithi187 ceph-mon[93536]: pgmap v515: 105 pgs: 105 active+clean; 4.9 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 324 KiB/s rd, 101 KiB/s wr, 486 op/s 2024-03-29T05:02:03.215 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:02 smithi187 ceph-mon[93536]: failed to encode map e564 with expected crc 2024-03-29T05:02:03.215 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:02 smithi187 ceph-mon[93536]: osdmap e564: 8 total, 8 up, 8 in 2024-03-29T05:02:03.215 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:02 smithi187 ceph-mon[93536]: failed to encode map e564 with expected crc 2024-03-29T05:02:03.215 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:02 smithi187 systemd[1]: Stopping Ceph osd.4 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:02:03.634 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[39028]: 2024-03-29T05:02:03.204+0000 7f0f11ee7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T05:02:03.635 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[39028]: 2024-03-29T05:02:03.204+0000 7f0f11ee7700 -1 osd.4 565 *** Got signal Terminated *** 2024-03-29T05:02:03.635 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[39028]: 2024-03-29T05:02:03.204+0000 7f0f11ee7700 -1 osd.4 565 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T05:02:04.143 INFO:teuthology.orchestra.run.smithi181.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T05:02:04.144 INFO:teuthology.orchestra.run.smithi181.stdout:alertmanager.a smithi181 *:9093,9094 running (7m) 12s ago 11m 25.4M - 0.25.0 c8568f914cd2 69e0ad67b57f 2024-03-29T05:02:04.144 INFO:teuthology.orchestra.run.smithi181.stdout:grafana.a smithi187 *:3000 error 12s ago 11m - - 2024-03-29T05:02:04.144 INFO:teuthology.orchestra.run.smithi181.stdout:mds.foo.smithi181.pflrwm smithi181 running (10m) 12s ago 10m 18.3M - 17.2.7-789-g4213e17c cf22eb8683be 0f200e33dce6 2024-03-29T05:02:04.144 INFO:teuthology.orchestra.run.smithi181.stdout:mds.foo.smithi187.vcsnqx smithi187 running (9m) 12s ago 9m 17.4M - 17.2.7-789-g4213e17c cf22eb8683be 245621ed14ed 2024-03-29T05:02:04.144 INFO:teuthology.orchestra.run.smithi181.stdout:mgr.x smithi187 *:8443,9283,8765 running (6m) 12s ago 15m 451M - 19.0.0-1959-g8da23624 54112f57da44 19bc315720bd 2024-03-29T05:02:04.144 INFO:teuthology.orchestra.run.smithi181.stdout:mgr.y smithi181 *:8443,9283,8765 running (6m) 12s ago 16m 509M - 19.0.0-1959-g8da23624 54112f57da44 b07d84f4244a 2024-03-29T05:02:04.144 INFO:teuthology.orchestra.run.smithi181.stdout:mon.a smithi181 running (6m) 12s ago 17m 62.6M 2048M 19.0.0-1959-g8da23624 54112f57da44 4e5d3325d991 2024-03-29T05:02:04.144 INFO:teuthology.orchestra.run.smithi181.stdout:mon.b smithi187 running (4m) 12s ago 15m 48.9M 2048M 19.0.0-1959-g8da23624 54112f57da44 4de0ae3c44c3 2024-03-29T05:02:04.144 INFO:teuthology.orchestra.run.smithi181.stdout:mon.c smithi181 running (4m) 12s ago 15m 55.3M 2048M 19.0.0-1959-g8da23624 54112f57da44 10f02d533230 2024-03-29T05:02:04.144 INFO:teuthology.orchestra.run.smithi181.stdout:node-exporter.a smithi181 *:9100 running (7m) 12s ago 10m 16.5M - 1.5.0 0da6a335fe13 4029d029600b 2024-03-29T05:02:04.144 INFO:teuthology.orchestra.run.smithi181.stdout:node-exporter.b smithi187 *:9100 running (6m) 12s ago 10m 15.5M - 1.5.0 0da6a335fe13 3894f6e97d55 2024-03-29T05:02:04.144 INFO:teuthology.orchestra.run.smithi181.stdout:osd.0 smithi181 running (2m) 12s ago 14m 73.1M 2480M 19.0.0-1959-g8da23624 54112f57da44 03d36b79ef26 2024-03-29T05:02:04.144 INFO:teuthology.orchestra.run.smithi181.stdout:osd.1 smithi181 running (86s) 12s ago 14m 60.6M 2480M 19.0.0-1959-g8da23624 54112f57da44 cb0b10fc92dd 2024-03-29T05:02:04.144 INFO:teuthology.orchestra.run.smithi181.stdout:osd.2 smithi181 running (52s) 12s ago 14m 57.0M 2480M 19.0.0-1959-g8da23624 54112f57da44 12c7ca8de226 2024-03-29T05:02:04.145 INFO:teuthology.orchestra.run.smithi181.stdout:osd.3 smithi181 running (17s) 12s ago 13m 11.8M 2480M 19.0.0-1959-g8da23624 54112f57da44 6ac673c0ed7c 2024-03-29T05:02:04.145 INFO:teuthology.orchestra.run.smithi181.stdout:osd.4 smithi187 running (13m) 12s ago 13m 140M 2480M 17.2.7-789-g4213e17c cf22eb8683be 42698769ef84 2024-03-29T05:02:04.145 INFO:teuthology.orchestra.run.smithi181.stdout:osd.5 smithi187 running (12m) 12s ago 12m 117M 2480M 17.2.7-789-g4213e17c cf22eb8683be 9599ff85a9e7 2024-03-29T05:02:04.145 INFO:teuthology.orchestra.run.smithi181.stdout:osd.6 smithi187 running (12m) 12s ago 12m 143M 2480M 17.2.7-789-g4213e17c cf22eb8683be c3287470c1fa 2024-03-29T05:02:04.145 INFO:teuthology.orchestra.run.smithi181.stdout:osd.7 smithi187 running (12m) 12s ago 12m 120M 2480M 17.2.7-789-g4213e17c cf22eb8683be 89fb8ccf8705 2024-03-29T05:02:04.145 INFO:teuthology.orchestra.run.smithi181.stdout:prometheus.a smithi187 *:9095 running (6m) 12s ago 11m 60.3M - 2.43.0 a07b618ecd1d 882b8922d74b 2024-03-29T05:02:04.170 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifyHeaderUpdate 2024-03-29T05:02:04.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:04 smithi181 ceph-mon[101970]: failed to encode map e565 with expected crc 2024-03-29T05:02:04.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:04 smithi181 ceph-mon[101970]: failed to encode map e565 with expected crc 2024-03-29T05:02:04.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:04 smithi181 ceph-mon[101970]: failed to encode map e565 with expected crc 2024-03-29T05:02:04.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:04 smithi181 ceph-mon[101970]: osdmap e565: 8 total, 8 up, 8 in 2024-03-29T05:02:04.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:04 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/462025232' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:04.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:04 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:04.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:04 smithi181 ceph-mon[101970]: osd.4 marked itself down and dead 2024-03-29T05:02:04.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:04 smithi181 ceph-mon[101970]: pgmap v518: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:02:04.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:04 smithi181 ceph-mon[106466]: failed to encode map e565 with expected crc 2024-03-29T05:02:04.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:04 smithi181 ceph-mon[106466]: failed to encode map e565 with expected crc 2024-03-29T05:02:04.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:04 smithi181 ceph-mon[106466]: failed to encode map e565 with expected crc 2024-03-29T05:02:04.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:04 smithi181 ceph-mon[106466]: osdmap e565: 8 total, 8 up, 8 in 2024-03-29T05:02:04.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:04 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/462025232' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:04.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:04 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:04.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:04 smithi181 ceph-mon[106466]: osd.4 marked itself down and dead 2024-03-29T05:02:04.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:04 smithi181 ceph-mon[106466]: pgmap v518: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:02:04.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:04 smithi187 ceph-mon[93536]: failed to encode map e565 with expected crc 2024-03-29T05:02:04.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:04 smithi187 ceph-mon[93536]: failed to encode map e565 with expected crc 2024-03-29T05:02:04.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:04 smithi187 ceph-mon[93536]: failed to encode map e565 with expected crc 2024-03-29T05:02:04.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:04 smithi187 ceph-mon[93536]: osdmap e565: 8 total, 8 up, 8 in 2024-03-29T05:02:04.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:04 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/462025232' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:04.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:04 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:04.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:04 smithi187 ceph-mon[93536]: osd.4 marked itself down and dead 2024-03-29T05:02:04.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:04 smithi187 ceph-mon[93536]: pgmap v518: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:02:04.749 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph versions' 2024-03-29T05:02:05.003 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:02:05.034 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:04 smithi187 podman[101077]: 2024-03-29 05:02:04.708578723 +0000 UTC m=+1.610401533 container died 42698769ef84744968cb3630a112f0d16630d1f1110d717e11476c8debeca177 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=quincy-4213e17) 2024-03-29T05:02:05.385 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:05 smithi187 podman[101077]: 2024-03-29 05:02:05.026551006 +0000 UTC m=+1.928373803 container cleanup 42698769ef84744968cb3630a112f0d16630d1f1110d717e11476c8debeca177 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17) 2024-03-29T05:02:05.450 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:05 smithi187 bash[101077]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4 2024-03-29T05:02:05.450 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:05 smithi187 podman[101100]: 2024-03-29 05:02:05.244777257 +0000 UTC m=+0.539410067 container remove 42698769ef84744968cb3630a112f0d16630d1f1110d717e11476c8debeca177 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=quincy-4213e17, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T05:02:05.450 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:05 smithi187 ceph-mon[93536]: from='client.44474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T05:02:05.451 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:05 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:02:05.451 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:05 smithi187 ceph-mon[93536]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T05:02:05.451 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:05 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:05.451 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:05 smithi187 ceph-mon[93536]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.451 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:05 smithi187 ceph-mon[93536]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.452 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:05 smithi187 ceph-mon[93536]: osdmap e566: 8 total, 7 up, 8 in 2024-03-29T05:02:05.452 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:05 smithi187 ceph-mon[93536]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.452 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:05 smithi187 ceph-mon[93536]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.452 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:05 smithi187 ceph-mon[93536]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.452 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:05 smithi187 ceph-mon[93536]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.452 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:05 smithi187 ceph-mon[93536]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.453 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:05 smithi187 ceph-mon[93536]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.453 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:05 smithi187 ceph-mon[93536]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[101970]: from='client.44474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T05:02:05.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:02:05.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[101970]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T05:02:05.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:05.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[101970]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[101970]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[101970]: osdmap e566: 8 total, 7 up, 8 in 2024-03-29T05:02:05.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[101970]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[101970]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[101970]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[101970]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[101970]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[101970]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[101970]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[106466]: from='client.44474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T05:02:05.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:02:05.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[106466]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T05:02:05.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:05.458 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[106466]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.458 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[106466]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.458 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[106466]: osdmap e566: 8 total, 7 up, 8 in 2024-03-29T05:02:05.458 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[106466]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.458 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[106466]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.459 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[106466]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.459 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[106466]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.459 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[106466]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.459 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[106466]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.459 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:05 smithi181 ceph-mon[106466]: failed to encode map e566 with expected crc 2024-03-29T05:02:05.954 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:05 smithi187 podman[101201]: 2024-03-29 05:02:05.589006244 +0000 UTC m=+0.018152403 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:02:06.205 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:06 smithi187 ceph-mon[93536]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.205 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:06 smithi187 ceph-mon[93536]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.205 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:06 smithi187 ceph-mon[93536]: osdmap e567: 8 total, 7 up, 8 in 2024-03-29T05:02:06.205 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:06 smithi187 ceph-mon[93536]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.205 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:06 smithi187 ceph-mon[93536]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.205 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:06 smithi187 ceph-mon[93536]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.205 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:06 smithi187 ceph-mon[93536]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.206 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:06 smithi187 ceph-mon[93536]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.206 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:06 smithi187 ceph-mon[93536]: pgmap v521: 105 pgs: 13 stale+active+clean, 32 unknown, 60 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:02:06.206 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:05 smithi187 podman[101201]: 2024-03-29 05:02:05.945846433 +0000 UTC m=+0.374992564 container create 1cbe904ca3da35396b1509ddbf8c1da63989a3c40637a1e5573ddf3f0aca5b28 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-deactivate, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-03-29T05:02:06.206 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:06 smithi187 podman[101201]: 2024-03-29 05:02:06.190667508 +0000 UTC m=+0.619813649 container init 1cbe904ca3da35396b1509ddbf8c1da63989a3c40637a1e5573ddf3f0aca5b28 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-03-29T05:02:06.421 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[106466]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.421 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[106466]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.422 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[106466]: osdmap e567: 8 total, 7 up, 8 in 2024-03-29T05:02:06.422 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[106466]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.422 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[106466]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.422 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[106466]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.422 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[106466]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.422 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[106466]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.422 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[106466]: pgmap v521: 105 pgs: 13 stale+active+clean, 32 unknown, 60 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:02:06.422 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[101970]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.422 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[101970]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.422 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[101970]: osdmap e567: 8 total, 7 up, 8 in 2024-03-29T05:02:06.422 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[101970]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.423 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[101970]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.423 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[101970]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.423 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[101970]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.423 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[101970]: failed to encode map e567 with expected crc 2024-03-29T05:02:06.423 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:06 smithi181 ceph-mon[101970]: pgmap v521: 105 pgs: 13 stale+active+clean, 32 unknown, 60 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:02:06.503 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:06 smithi187 podman[101201]: 2024-03-29 05:02:06.196779072 +0000 UTC m=+0.625925204 container start 1cbe904ca3da35396b1509ddbf8c1da63989a3c40637a1e5573ddf3f0aca5b28 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T05:02:06.503 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:06 smithi187 podman[101201]: 2024-03-29 05:02:06.298850595 +0000 UTC m=+0.727996739 container attach 1cbe904ca3da35396b1509ddbf8c1da63989a3c40637a1e5573ddf3f0aca5b28 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T05:02:06.843 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:06 smithi187 podman[101201]: 2024-03-29 05:02:06.494917688 +0000 UTC m=+0.924063831 container died 1cbe904ca3da35396b1509ddbf8c1da63989a3c40637a1e5573ddf3f0aca5b28 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-deactivate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-03-29T05:02:06.911 INFO:teuthology.orchestra.run.smithi181.stdout:{ 2024-03-29T05:02:06.911 INFO:teuthology.orchestra.run.smithi181.stdout: "mon": { 2024-03-29T05:02:06.911 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 3 2024-03-29T05:02:06.911 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2024-03-29T05:02:06.911 INFO:teuthology.orchestra.run.smithi181.stdout: "mgr": { 2024-03-29T05:02:06.911 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 2 2024-03-29T05:02:06.911 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2024-03-29T05:02:06.911 INFO:teuthology.orchestra.run.smithi181.stdout: "osd": { 2024-03-29T05:02:06.911 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)": 3, 2024-03-29T05:02:06.911 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 4 2024-03-29T05:02:06.911 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2024-03-29T05:02:06.912 INFO:teuthology.orchestra.run.smithi181.stdout: "mds": { 2024-03-29T05:02:06.912 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)": 2 2024-03-29T05:02:06.912 INFO:teuthology.orchestra.run.smithi181.stdout: }, 2024-03-29T05:02:06.912 INFO:teuthology.orchestra.run.smithi181.stdout: "overall": { 2024-03-29T05:02:06.912 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)": 5, 2024-03-29T05:02:06.912 INFO:teuthology.orchestra.run.smithi181.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 9 2024-03-29T05:02:06.912 INFO:teuthology.orchestra.run.smithi181.stdout: } 2024-03-29T05:02:06.912 INFO:teuthology.orchestra.run.smithi181.stdout:} 2024-03-29T05:02:07.205 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifyHeaderUpdate (3035 ms) 2024-03-29T05:02:07.205 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifyFlatten 2024-03-29T05:02:07.228 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifyFlatten (23 ms) 2024-03-29T05:02:07.228 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifyResize 2024-03-29T05:02:07.248 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifyResize (21 ms) 2024-03-29T05:02:07.249 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifyRebuildObjectMap 2024-03-29T05:02:07.267 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifyRebuildObjectMap (19 ms) 2024-03-29T05:02:07.267 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifyUpdateFeatures 2024-03-29T05:02:07.286 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifyUpdateFeatures (18 ms) 2024-03-29T05:02:07.286 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifySnapCreate 2024-03-29T05:02:07.307 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifySnapCreate (21 ms) 2024-03-29T05:02:07.307 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifySnapCreateError 2024-03-29T05:02:07.325 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifySnapCreateError (19 ms) 2024-03-29T05:02:07.326 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifySnapRename 2024-03-29T05:02:07.346 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifySnapRename (20 ms) 2024-03-29T05:02:07.346 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifySnapRenameError 2024-03-29T05:02:07.365 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifySnapRenameError (19 ms) 2024-03-29T05:02:07.365 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifySnapRemove 2024-03-29T05:02:07.384 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifySnapRemove (19 ms) 2024-03-29T05:02:07.384 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifySnapProtect 2024-03-29T05:02:07.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:07 smithi187 ceph-mon[93536]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:02:07.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:07 smithi187 ceph-mon[93536]: failed to encode map e568 with expected crc 2024-03-29T05:02:07.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:07 smithi187 ceph-mon[93536]: osdmap e568: 8 total, 7 up, 8 in 2024-03-29T05:02:07.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:07 smithi187 ceph-mon[93536]: failed to encode map e568 with expected crc 2024-03-29T05:02:07.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:07 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:07.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:07 smithi187 ceph-mon[93536]: failed to encode map e568 with expected crc 2024-03-29T05:02:07.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:07 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3063985549' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:02:07.385 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:07 smithi187 podman[101248]: 2024-03-29 05:02:07.093993457 +0000 UTC m=+0.593591641 container remove 1cbe904ca3da35396b1509ddbf8c1da63989a3c40637a1e5573ddf3f0aca5b28 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-deactivate, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-8da2362) 2024-03-29T05:02:07.385 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:07 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.4.service: Deactivated successfully. 2024-03-29T05:02:07.385 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:07 smithi187 systemd[1]: Stopped Ceph osd.4 for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T05:02:07.386 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:07 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.4.service: Consumed 26.777s CPU time. 2024-03-29T05:02:07.386 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:07 smithi187 systemd[1]: Starting Ceph osd.4 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:02:07.404 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifySnapProtect (20 ms) 2024-03-29T05:02:07.404 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifySnapUnprotect 2024-03-29T05:02:07.424 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifySnapUnprotect (20 ms) 2024-03-29T05:02:07.424 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifyRename 2024-03-29T05:02:07.445 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifyRename (21 ms) 2024-03-29T05:02:07.445 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifyAsyncTimedOut 2024-03-29T05:02:07.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:07 smithi181 ceph-mon[106466]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:02:07.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:07 smithi181 ceph-mon[106466]: failed to encode map e568 with expected crc 2024-03-29T05:02:07.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:07 smithi181 ceph-mon[106466]: osdmap e568: 8 total, 7 up, 8 in 2024-03-29T05:02:07.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:07 smithi181 ceph-mon[106466]: failed to encode map e568 with expected crc 2024-03-29T05:02:07.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:07 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:07.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:07 smithi181 ceph-mon[106466]: failed to encode map e568 with expected crc 2024-03-29T05:02:07.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:07 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3063985549' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:02:07.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:07 smithi181 ceph-mon[101970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:02:07.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:07 smithi181 ceph-mon[101970]: failed to encode map e568 with expected crc 2024-03-29T05:02:07.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:07 smithi181 ceph-mon[101970]: osdmap e568: 8 total, 7 up, 8 in 2024-03-29T05:02:07.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:07 smithi181 ceph-mon[101970]: failed to encode map e568 with expected crc 2024-03-29T05:02:07.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:07 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:07.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:07 smithi181 ceph-mon[101970]: failed to encode map e568 with expected crc 2024-03-29T05:02:07.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:07 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3063985549' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T05:02:07.463 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifyAsyncTimedOut (19 ms) 2024-03-29T05:02:07.464 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifyAsyncError 2024-03-29T05:02:07.482 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifyAsyncError (19 ms) 2024-03-29T05:02:07.482 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifyAsyncCompleteError 2024-03-29T05:02:07.501 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifyAsyncCompleteError (19 ms) 2024-03-29T05:02:07.502 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestImageWatcher.NotifyAsyncRequestTimedOut 2024-03-29T05:02:07.521 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestImageWatcher.NotifyAsyncRequestTimedOut (19 ms) 2024-03-29T05:02:07.657 DEBUG:teuthology.parallel:result is None 2024-03-29T05:02:07.831 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:07 smithi187 podman[101355]: 2024-03-29 05:02:07.463061244 +0000 UTC m=+0.018604968 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:02:07.831 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:07 smithi187 podman[101355]: 2024-03-29 05:02:07.578378132 +0000 UTC m=+0.133921857 container create 8e011f232cf561ea701b7ea601e3e51ca54998190601c72aa4d871c38f791e42 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-activate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362) 2024-03-29T05:02:07.831 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:07 smithi187 podman[101355]: 2024-03-29 05:02:07.822778245 +0000 UTC m=+0.378321975 container init 8e011f232cf561ea701b7ea601e3e51ca54998190601c72aa4d871c38f791e42 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-activate, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-03-29T05:02:08.135 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:07 smithi187 podman[101355]: 2024-03-29 05:02:07.828694108 +0000 UTC m=+0.384237833 container start 8e011f232cf561ea701b7ea601e3e51ca54998190601c72aa4d871c38f791e42 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2) 2024-03-29T05:02:08.135 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:07 smithi187 podman[101355]: 2024-03-29 05:02:07.914236763 +0000 UTC m=+0.469780494 container attach 8e011f232cf561ea701b7ea601e3e51ca54998190601c72aa4d871c38f791e42 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362) 2024-03-29T05:02:08.197 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 17 tests from TestImageWatcher (3351 ms total) 2024-03-29T05:02:08.197 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:02:08.197 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 159 tests from TestLibRBD 2024-03-29T05:02:08.225 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.CreateAndStat 2024-03-29T05:02:08.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:08 smithi181 ceph-mon[101970]: failed to encode map e569 with expected crc 2024-03-29T05:02:08.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:08 smithi181 ceph-mon[101970]: osdmap e569: 8 total, 7 up, 8 in 2024-03-29T05:02:08.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:08 smithi181 ceph-mon[101970]: failed to encode map e569 with expected crc 2024-03-29T05:02:08.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:08 smithi181 ceph-mon[101970]: failed to encode map e569 with expected crc 2024-03-29T05:02:08.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:08 smithi181 ceph-mon[101970]: failed to encode map e569 with expected crc 2024-03-29T05:02:08.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:08 smithi181 ceph-mon[101970]: pgmap v524: 105 pgs: 9 active+undersized+degraded, 36 active+undersized, 60 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 14/75 objects degraded (18.667%) 2024-03-29T05:02:08.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:08 smithi181 ceph-mon[106466]: failed to encode map e569 with expected crc 2024-03-29T05:02:08.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:08 smithi181 ceph-mon[106466]: osdmap e569: 8 total, 7 up, 8 in 2024-03-29T05:02:08.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:08 smithi181 ceph-mon[106466]: failed to encode map e569 with expected crc 2024-03-29T05:02:08.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:08 smithi181 ceph-mon[106466]: failed to encode map e569 with expected crc 2024-03-29T05:02:08.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:08 smithi181 ceph-mon[106466]: failed to encode map e569 with expected crc 2024-03-29T05:02:08.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:08 smithi181 ceph-mon[106466]: pgmap v524: 105 pgs: 9 active+undersized+degraded, 36 active+undersized, 60 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 14/75 objects degraded (18.667%) 2024-03-29T05:02:08.594 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:08 smithi187 ceph-mon[93536]: failed to encode map e569 with expected crc 2024-03-29T05:02:08.594 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:08 smithi187 ceph-mon[93536]: osdmap e569: 8 total, 7 up, 8 in 2024-03-29T05:02:08.594 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:08 smithi187 ceph-mon[93536]: failed to encode map e569 with expected crc 2024-03-29T05:02:08.594 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:08 smithi187 ceph-mon[93536]: failed to encode map e569 with expected crc 2024-03-29T05:02:08.594 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:08 smithi187 ceph-mon[93536]: failed to encode map e569 with expected crc 2024-03-29T05:02:08.594 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:08 smithi187 ceph-mon[93536]: pgmap v524: 105 pgs: 9 active+undersized+degraded, 36 active+undersized, 60 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 14/75 objects degraded (18.667%) 2024-03-29T05:02:08.595 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-activate[101369]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-29T05:02:08.595 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-activate[101369]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-29T05:02:08.595 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:08 smithi187 bash[101355]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-29T05:02:08.595 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:08 smithi187 bash[101355]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-29T05:02:08.595 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:08 smithi187 bash[101355]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-29T05:02:08.595 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:08 smithi187 bash[101355]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-29T05:02:08.595 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:08 smithi187 bash[101355]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-03-29T05:02:08.595 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:08 smithi187 bash[101355]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-29T05:02:08.595 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:08 smithi187 bash[101355]: --> ceph-volume raw activate successful for osd ID: 4 2024-03-29T05:02:08.595 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-activate[101369]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-29T05:02:08.595 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-activate[101369]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-29T05:02:08.595 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-activate[101369]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-03-29T05:02:08.595 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-activate[101369]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-29T05:02:08.596 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-activate[101369]: --> ceph-volume raw activate successful for osd ID: 4 2024-03-29T05:02:08.596 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:08 smithi187 podman[101355]: 2024-03-29 05:02:08.295678856 +0000 UTC m=+0.851222571 container died 8e011f232cf561ea701b7ea601e3e51ca54998190601c72aa4d871c38f791e42 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-activate, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-03-29T05:02:08.884 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:08 smithi187 podman[101519]: 2024-03-29 05:02:08.800718607 +0000 UTC m=+0.499921435 container remove 8e011f232cf561ea701b7ea601e3e51ca54998190601c72aa4d871c38f791e42 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-activate, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0) 2024-03-29T05:02:09.304 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:09 smithi187 ceph-mon[93536]: Health check failed: Degraded data redundancy: 14/75 objects degraded (18.667%), 9 pgs degraded (PG_DEGRADED) 2024-03-29T05:02:09.304 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:09 smithi187 ceph-mon[93536]: failed to encode map e570 with expected crc 2024-03-29T05:02:09.304 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:09 smithi187 ceph-mon[93536]: osdmap e570: 8 total, 7 up, 8 in 2024-03-29T05:02:09.304 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:09 smithi187 podman[101589]: 2024-03-29 05:02:08.920344324 +0000 UTC m=+0.018374243 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:02:09.304 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:09 smithi187 podman[101589]: 2024-03-29 05:02:09.051732969 +0000 UTC m=+0.149762888 container create 49e5104e4a5d385928a561b1da11f7dae432886672d933441b9e9da4f5cb2716 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, RELEASE=squid-8da2362, io.buildah.version=1.33.5) 2024-03-29T05:02:09.304 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:09 smithi187 podman[101589]: 2024-03-29 05:02:09.295808879 +0000 UTC m=+0.393838804 container init 49e5104e4a5d385928a561b1da11f7dae432886672d933441b9e9da4f5cb2716 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-03-29T05:02:09.385 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:02:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:02:08] "GET /metrics HTTP/1.1" 200 36326 "" "Prometheus/2.43.0" 2024-03-29T05:02:09.385 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:09 smithi181 ceph-mon[106466]: Health check failed: Degraded data redundancy: 14/75 objects degraded (18.667%), 9 pgs degraded (PG_DEGRADED) 2024-03-29T05:02:09.386 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:09 smithi181 ceph-mon[106466]: failed to encode map e570 with expected crc 2024-03-29T05:02:09.386 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:09 smithi181 ceph-mon[106466]: osdmap e570: 8 total, 7 up, 8 in 2024-03-29T05:02:09.386 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:09 smithi181 ceph-mon[101970]: Health check failed: Degraded data redundancy: 14/75 objects degraded (18.667%), 9 pgs degraded (PG_DEGRADED) 2024-03-29T05:02:09.386 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:09 smithi181 ceph-mon[101970]: failed to encode map e570 with expected crc 2024-03-29T05:02:09.386 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:09 smithi181 ceph-mon[101970]: osdmap e570: 8 total, 7 up, 8 in 2024-03-29T05:02:09.635 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:09 smithi187 podman[101589]: 2024-03-29 05:02:09.301303622 +0000 UTC m=+0.399333545 container start 49e5104e4a5d385928a561b1da11f7dae432886672d933441b9e9da4f5cb2716 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-8da2362) 2024-03-29T05:02:09.635 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:09 smithi187 ceph-osd[101627]: -- 172.21.15.187:0/1178967587 <== mon.1 v2:172.21.15.187:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55efdf037680 con 0x55efdf001000 2024-03-29T05:02:09.635 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:09 smithi187 bash[101589]: 49e5104e4a5d385928a561b1da11f7dae432886672d933441b9e9da4f5cb2716 2024-03-29T05:02:09.635 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:09 smithi187 systemd[1]: Started Ceph osd.4 for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T05:02:10.135 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:09 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:02:09.838+0000 7f69f82697c0 -1 Falling back to public interface 2024-03-29T05:02:10.208 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:10.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:10 smithi181 ceph-mon[106466]: failed to encode map e571 with expected crc 2024-03-29T05:02:10.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:10 smithi181 ceph-mon[106466]: failed to encode map e571 with expected crc 2024-03-29T05:02:10.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:10 smithi181 ceph-mon[106466]: osdmap e571: 8 total, 7 up, 8 in 2024-03-29T05:02:10.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:10 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2976588493' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:10.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:10 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:10.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:10 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:10.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:10 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:02:10.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:10 smithi181 ceph-mon[106466]: pgmap v527: 105 pgs: 32 unknown, 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-03-29T05:02:10.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:10 smithi181 ceph-mon[101970]: failed to encode map e571 with expected crc 2024-03-29T05:02:10.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:10 smithi181 ceph-mon[101970]: failed to encode map e571 with expected crc 2024-03-29T05:02:10.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:10 smithi181 ceph-mon[101970]: osdmap e571: 8 total, 7 up, 8 in 2024-03-29T05:02:10.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:10 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2976588493' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:10.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:10 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:10.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:10 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:10.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:10 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:02:10.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:10 smithi181 ceph-mon[101970]: pgmap v527: 105 pgs: 32 unknown, 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-03-29T05:02:10.520 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:10 smithi187 ceph-mon[93536]: failed to encode map e571 with expected crc 2024-03-29T05:02:10.521 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:10 smithi187 ceph-mon[93536]: failed to encode map e571 with expected crc 2024-03-29T05:02:10.521 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:10 smithi187 ceph-mon[93536]: osdmap e571: 8 total, 7 up, 8 in 2024-03-29T05:02:10.521 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:10 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2976588493' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:10.521 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:10 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:10.521 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:10 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:10.521 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:10 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:02:10.521 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:10 smithi187 ceph-mon[93536]: pgmap v527: 105 pgs: 32 unknown, 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2024-03-29T05:02:11.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:11 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2976588493' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:11.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:11 smithi181 ceph-mon[101970]: failed to encode map e572 with expected crc 2024-03-29T05:02:11.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:11 smithi181 ceph-mon[101970]: osdmap e572: 8 total, 7 up, 8 in 2024-03-29T05:02:11.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:11 smithi181 ceph-mon[101970]: failed to encode map e572 with expected crc 2024-03-29T05:02:11.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:11 smithi181 ceph-mon[101970]: failed to encode map e572 with expected crc 2024-03-29T05:02:11.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:11 smithi181 ceph-mon[101970]: failed to encode map e572 with expected crc 2024-03-29T05:02:11.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:11 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:02:11.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:11 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2976588493' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:11.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:11 smithi181 ceph-mon[106466]: failed to encode map e572 with expected crc 2024-03-29T05:02:11.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:11 smithi181 ceph-mon[106466]: osdmap e572: 8 total, 7 up, 8 in 2024-03-29T05:02:11.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:11 smithi181 ceph-mon[106466]: failed to encode map e572 with expected crc 2024-03-29T05:02:11.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:11 smithi181 ceph-mon[106466]: failed to encode map e572 with expected crc 2024-03-29T05:02:11.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:11 smithi181 ceph-mon[106466]: failed to encode map e572 with expected crc 2024-03-29T05:02:11.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:11 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:02:11.619 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:11 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2976588493' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:11.619 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:11 smithi187 ceph-mon[93536]: failed to encode map e572 with expected crc 2024-03-29T05:02:11.619 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:11 smithi187 ceph-mon[93536]: osdmap e572: 8 total, 7 up, 8 in 2024-03-29T05:02:11.619 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:11 smithi187 ceph-mon[93536]: failed to encode map e572 with expected crc 2024-03-29T05:02:11.619 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:11 smithi187 ceph-mon[93536]: failed to encode map e572 with expected crc 2024-03-29T05:02:11.619 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:11 smithi187 ceph-mon[93536]: failed to encode map e572 with expected crc 2024-03-29T05:02:11.619 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:11 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:02:12.242 INFO:tasks.workunit.client.0.smithi181.stdout:image has size 2097152 and order 22 2024-03-29T05:02:12.243 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.CreateAndStat (4017 ms) 2024-03-29T05:02:12.243 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.CreateWithSameDataPool 2024-03-29T05:02:12.243 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:12.258 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.CreateWithSameDataPool (16 ms) 2024-03-29T05:02:12.259 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.CreateAndStatPP 2024-03-29T05:02:12.259 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:12.277 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.CreateAndStatPP (18 ms) 2024-03-29T05:02:12.277 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.GetId 2024-03-29T05:02:12.277 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:12.306 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.GetId (29 ms) 2024-03-29T05:02:12.306 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.GetIdPP 2024-03-29T05:02:12.307 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:12.329 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.GetIdPP (23 ms) 2024-03-29T05:02:12.329 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefix 2024-03-29T05:02:12.329 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:12.345 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.GetBlockNamePrefix (16 ms) 2024-03-29T05:02:12.345 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefixPP 2024-03-29T05:02:12.345 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:12.359 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.GetBlockNamePrefixPP (15 ms) 2024-03-29T05:02:12.359 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestGetCreateTimestamp 2024-03-29T05:02:12.360 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:12.374 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestGetCreateTimestamp (15 ms) 2024-03-29T05:02:12.374 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.GetCreateTimestampPP 2024-03-29T05:02:12.375 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:12.393 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.GetCreateTimestampPP (18 ms) 2024-03-29T05:02:12.393 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.OpenAio 2024-03-29T05:02:12.393 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:12.411 INFO:tasks.workunit.client.0.smithi181.stdout:image has size 2097152 and order 22 2024-03-29T05:02:12.411 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.OpenAio (18 ms) 2024-03-29T05:02:12.411 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.OpenAioFail 2024-03-29T05:02:12.413 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.OpenAioFail (3 ms) 2024-03-29T05:02:12.413 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.OpenAioPP 2024-03-29T05:02:12.413 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:12.436 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.OpenAioPP (22 ms) 2024-03-29T05:02:12.436 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.OpenAioFailPP 2024-03-29T05:02:12.439 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.OpenAioFailPP (3 ms) 2024-03-29T05:02:12.439 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ResizeAndStat 2024-03-29T05:02:12.439 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:12.471 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ResizeAndStat (32 ms) 2024-03-29T05:02:12.471 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ResizeAndStatPP 2024-03-29T05:02:12.471 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:12.502 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ResizeAndStatPP (31 ms) 2024-03-29T05:02:12.502 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResize 2024-03-29T05:02:12.502 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:12.533 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.UpdateWatchAndResize (31 ms) 2024-03-29T05:02:12.533 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResizePP 2024-03-29T05:02:12.533 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:12.565 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.UpdateWatchAndResizePP (32 ms) 2024-03-29T05:02:12.565 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCreateLsDelete 2024-03-29T05:02:12.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:12 smithi187 ceph-mon[93536]: failed to encode map e573 with expected crc 2024-03-29T05:02:12.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:12 smithi187 ceph-mon[93536]: osdmap e573: 8 total, 7 up, 8 in 2024-03-29T05:02:12.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:12 smithi187 ceph-mon[93536]: failed to encode map e573 with expected crc 2024-03-29T05:02:12.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:12.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:02:12.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:12 smithi187 ceph-mon[93536]: pgmap v530: 105 pgs: 16 unknown, 10 active+undersized+degraded, 30 active+undersized, 49 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15/75 objects degraded (20.000%) 2024-03-29T05:02:12.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:12 smithi181 ceph-mon[106466]: failed to encode map e573 with expected crc 2024-03-29T05:02:12.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:12 smithi181 ceph-mon[106466]: osdmap e573: 8 total, 7 up, 8 in 2024-03-29T05:02:12.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:12 smithi181 ceph-mon[106466]: failed to encode map e573 with expected crc 2024-03-29T05:02:12.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:12.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:02:12.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:12 smithi181 ceph-mon[106466]: pgmap v530: 105 pgs: 16 unknown, 10 active+undersized+degraded, 30 active+undersized, 49 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15/75 objects degraded (20.000%) 2024-03-29T05:02:12.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:12 smithi181 ceph-mon[101970]: failed to encode map e573 with expected crc 2024-03-29T05:02:12.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:12 smithi181 ceph-mon[101970]: osdmap e573: 8 total, 7 up, 8 in 2024-03-29T05:02:12.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:12 smithi181 ceph-mon[101970]: failed to encode map e573 with expected crc 2024-03-29T05:02:12.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:12.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:02:12.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:12 smithi181 ceph-mon[101970]: pgmap v530: 105 pgs: 16 unknown, 10 active+undersized+degraded, 30 active+undersized, 49 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15/75 objects degraded (20.000%) 2024-03-29T05:02:13.135 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:12 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:02:12.671+0000 7f69f82697c0 -1 osd.4 565 log_to_monitors true 2024-03-29T05:02:13.635 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:02:13 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:02:13.265+0000 7f69e96c1700 -1 osd.4 565 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T05:02:13.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:13 smithi187 ceph-mon[93536]: failed to encode map e574 with expected crc 2024-03-29T05:02:13.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:13 smithi187 ceph-mon[93536]: osdmap e574: 8 total, 7 up, 8 in 2024-03-29T05:02:13.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:13 smithi187 ceph-mon[93536]: failed to encode map e574 with expected crc 2024-03-29T05:02:13.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:13 smithi187 ceph-mon[93536]: failed to encode map e574 with expected crc 2024-03-29T05:02:13.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:13 smithi187 ceph-mon[93536]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:02:13.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:13 smithi187 ceph-mon[93536]: from='osd.4 [v2:172.21.15.187:6800/1357613329,v1:172.21.15.187:6801/1357613329]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T05:02:13.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:13 smithi187 ceph-mon[93536]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T05:02:13.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:13 smithi181 ceph-mon[106466]: failed to encode map e574 with expected crc 2024-03-29T05:02:13.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:13 smithi181 ceph-mon[106466]: osdmap e574: 8 total, 7 up, 8 in 2024-03-29T05:02:13.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:13 smithi181 ceph-mon[106466]: failed to encode map e574 with expected crc 2024-03-29T05:02:13.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:13 smithi181 ceph-mon[106466]: failed to encode map e574 with expected crc 2024-03-29T05:02:13.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:13 smithi181 ceph-mon[106466]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:02:13.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:13 smithi181 ceph-mon[106466]: from='osd.4 [v2:172.21.15.187:6800/1357613329,v1:172.21.15.187:6801/1357613329]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T05:02:13.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:13 smithi181 ceph-mon[106466]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T05:02:13.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:13 smithi181 ceph-mon[101970]: failed to encode map e574 with expected crc 2024-03-29T05:02:13.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:13 smithi181 ceph-mon[101970]: osdmap e574: 8 total, 7 up, 8 in 2024-03-29T05:02:13.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:13 smithi181 ceph-mon[101970]: failed to encode map e574 with expected crc 2024-03-29T05:02:13.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:13 smithi181 ceph-mon[101970]: failed to encode map e574 with expected crc 2024-03-29T05:02:13.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:13 smithi181 ceph-mon[101970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:02:13.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:13 smithi181 ceph-mon[101970]: from='osd.4 [v2:172.21.15.187:6800/1357613329,v1:172.21.15.187:6801/1357613329]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T05:02:13.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:13 smithi181 ceph-mon[101970]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T05:02:14.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:14 smithi187 ceph-mon[93536]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-29T05:02:14.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:14 smithi187 ceph-mon[93536]: osdmap e575: 8 total, 7 up, 8 in 2024-03-29T05:02:14.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:14 smithi187 ceph-mon[93536]: from='osd.4 [v2:172.21.15.187:6800/1357613329,v1:172.21.15.187:6801/1357613329]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T05:02:14.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:14 smithi187 ceph-mon[93536]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:14 smithi187 ceph-mon[93536]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:14 smithi187 ceph-mon[93536]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:14 smithi187 ceph-mon[93536]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:14 smithi187 ceph-mon[93536]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:14 smithi187 ceph-mon[93536]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T05:02:14.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:14 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2056193417' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:14.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:14 smithi187 ceph-mon[93536]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:14 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:14.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:14 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:14.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:14 smithi187 ceph-mon[93536]: pgmap v533: 137 pgs: 32 unknown, 10 active+undersized+degraded, 35 active+undersized, 60 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 15/75 objects degraded (20.000%) 2024-03-29T05:02:14.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[106466]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-29T05:02:14.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[106466]: osdmap e575: 8 total, 7 up, 8 in 2024-03-29T05:02:14.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[106466]: from='osd.4 [v2:172.21.15.187:6800/1357613329,v1:172.21.15.187:6801/1357613329]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T05:02:14.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[106466]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[106466]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[106466]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[106466]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[106466]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[106466]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T05:02:14.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2056193417' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:14.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[106466]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:14.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:14.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[106466]: pgmap v533: 137 pgs: 32 unknown, 10 active+undersized+degraded, 35 active+undersized, 60 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 15/75 objects degraded (20.000%) 2024-03-29T05:02:14.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[101970]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-29T05:02:14.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[101970]: osdmap e575: 8 total, 7 up, 8 in 2024-03-29T05:02:14.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[101970]: from='osd.4 [v2:172.21.15.187:6800/1357613329,v1:172.21.15.187:6801/1357613329]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T05:02:14.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[101970]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[101970]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[101970]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[101970]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[101970]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[101970]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi187", "root=default"]}]: dispatch 2024-03-29T05:02:14.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2056193417' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:14.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[101970]: failed to encode map e575 with expected crc 2024-03-29T05:02:14.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:14.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:14.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:14 smithi181 ceph-mon[101970]: pgmap v533: 137 pgs: 32 unknown, 10 active+undersized+degraded, 35 active+undersized, 60 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s; 15/75 objects degraded (20.000%) 2024-03-29T05:02:15.270 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:15.512 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:15 smithi187 ceph-mon[93536]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T05:02:15.512 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:15 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2056193417' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:15.512 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:15 smithi187 ceph-mon[93536]: failed to encode map e576 with expected crc 2024-03-29T05:02:15.512 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:15 smithi187 ceph-mon[93536]: osd.4 [v2:172.21.15.187:6800/1357613329,v1:172.21.15.187:6801/1357613329] boot 2024-03-29T05:02:15.512 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:15 smithi187 ceph-mon[93536]: osdmap e576: 8 total, 8 up, 8 in 2024-03-29T05:02:15.512 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:15 smithi187 ceph-mon[93536]: failed to encode map e576 with expected crc 2024-03-29T05:02:15.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:15 smithi187 ceph-mon[93536]: failed to encode map e576 with expected crc 2024-03-29T05:02:15.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:15 smithi187 ceph-mon[93536]: failed to encode map e576 with expected crc 2024-03-29T05:02:15.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:15 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T05:02:15.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:15 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:02:15.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:15 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:15.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:15 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:15.513 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:15 smithi187 ceph-mon[93536]: osdmap e577: 8 total, 8 up, 8 in 2024-03-29T05:02:15.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[106466]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T05:02:15.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2056193417' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:15.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[106466]: failed to encode map e576 with expected crc 2024-03-29T05:02:15.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[106466]: osd.4 [v2:172.21.15.187:6800/1357613329,v1:172.21.15.187:6801/1357613329] boot 2024-03-29T05:02:15.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[106466]: osdmap e576: 8 total, 8 up, 8 in 2024-03-29T05:02:15.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[106466]: failed to encode map e576 with expected crc 2024-03-29T05:02:15.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[106466]: failed to encode map e576 with expected crc 2024-03-29T05:02:15.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[106466]: failed to encode map e576 with expected crc 2024-03-29T05:02:15.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T05:02:15.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:02:15.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:15.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:15.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[106466]: osdmap e577: 8 total, 8 up, 8 in 2024-03-29T05:02:15.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[101970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T05:02:15.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2056193417' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:15.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[101970]: failed to encode map e576 with expected crc 2024-03-29T05:02:15.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[101970]: osd.4 [v2:172.21.15.187:6800/1357613329,v1:172.21.15.187:6801/1357613329] boot 2024-03-29T05:02:15.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[101970]: osdmap e576: 8 total, 8 up, 8 in 2024-03-29T05:02:15.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[101970]: failed to encode map e576 with expected crc 2024-03-29T05:02:15.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[101970]: failed to encode map e576 with expected crc 2024-03-29T05:02:15.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[101970]: failed to encode map e576 with expected crc 2024-03-29T05:02:15.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T05:02:15.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:02:15.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:15.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:15.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:15 smithi181 ceph-mon[101970]: osdmap e577: 8 total, 8 up, 8 in 2024-03-29T05:02:16.584 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:16 smithi187 ceph-mon[93536]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.585 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:16 smithi187 ceph-mon[93536]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.585 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:16 smithi187 ceph-mon[93536]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.585 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:16 smithi187 ceph-mon[93536]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.585 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:16 smithi187 ceph-mon[93536]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.585 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:16 smithi187 ceph-mon[93536]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.585 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:16 smithi187 ceph-mon[93536]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.585 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:16 smithi187 ceph-mon[93536]: pgmap v536: 137 pgs: 32 unknown, 10 active+undersized+degraded, 35 active+undersized, 60 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15/75 objects degraded (20.000%) 2024-03-29T05:02:16.585 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:16 smithi187 ceph-mon[93536]: osdmap e578: 8 total, 8 up, 8 in 2024-03-29T05:02:16.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[101970]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[101970]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[101970]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[101970]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[101970]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[101970]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[101970]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[101970]: pgmap v536: 137 pgs: 32 unknown, 10 active+undersized+degraded, 35 active+undersized, 60 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15/75 objects degraded (20.000%) 2024-03-29T05:02:16.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[101970]: osdmap e578: 8 total, 8 up, 8 in 2024-03-29T05:02:16.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[106466]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[106466]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[106466]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[106466]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[106466]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[106466]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[106466]: failed to encode map e577 with expected crc 2024-03-29T05:02:16.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[106466]: pgmap v536: 137 pgs: 32 unknown, 10 active+undersized+degraded, 35 active+undersized, 60 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15/75 objects degraded (20.000%) 2024-03-29T05:02:16.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:16 smithi181 ceph-mon[106466]: osdmap e578: 8 total, 8 up, 8 in 2024-03-29T05:02:17.263 INFO:tasks.workunit.client.0.smithi181.stdout:image: image18 2024-03-29T05:02:17.263 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image18 2024-03-29T05:02:17.263 INFO:tasks.workunit.client.0.smithi181.stdout:found image18 2024-03-29T05:02:17.263 INFO:tasks.workunit.client.0.smithi181.stdout:erased image18 2024-03-29T05:02:17.263 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:17.342 INFO:tasks.workunit.client.0.smithi181.stdout:image: image18 2024-03-29T05:02:17.342 INFO:tasks.workunit.client.0.smithi181.stdout:image: image19 2024-03-29T05:02:17.342 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image18 2024-03-29T05:02:17.342 INFO:tasks.workunit.client.0.smithi181.stdout:found image18 2024-03-29T05:02:17.342 INFO:tasks.workunit.client.0.smithi181.stdout:erased image18 2024-03-29T05:02:17.342 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image19 2024-03-29T05:02:17.343 INFO:tasks.workunit.client.0.smithi181.stdout:found image19 2024-03-29T05:02:17.343 INFO:tasks.workunit.client.0.smithi181.stdout:erased image19 2024-03-29T05:02:17.343 INFO:tasks.workunit.client.0.smithi181.stdout:image: image19 2024-03-29T05:02:17.343 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image19 2024-03-29T05:02:17.343 INFO:tasks.workunit.client.0.smithi181.stdout:found image19 2024-03-29T05:02:17.343 INFO:tasks.workunit.client.0.smithi181.stdout:erased image19 2024-03-29T05:02:17.343 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCreateLsDelete (4777 ms) 2024-03-29T05:02:17.343 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCreateLsDeletePP 2024-03-29T05:02:17.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:17 smithi187 ceph-mon[93536]: failed to encode map e578 with expected crc 2024-03-29T05:02:17.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:17 smithi187 ceph-mon[93536]: failed to encode map e578 with expected crc 2024-03-29T05:02:17.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:17 smithi187 ceph-mon[93536]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:02:17.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:17 smithi187 ceph-mon[93536]: osdmap e579: 8 total, 8 up, 8 in 2024-03-29T05:02:17.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:17 smithi181 ceph-mon[106466]: failed to encode map e578 with expected crc 2024-03-29T05:02:17.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:17 smithi181 ceph-mon[106466]: failed to encode map e578 with expected crc 2024-03-29T05:02:17.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:17 smithi181 ceph-mon[106466]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:02:17.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:17 smithi181 ceph-mon[106466]: osdmap e579: 8 total, 8 up, 8 in 2024-03-29T05:02:17.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:17 smithi181 ceph-mon[101970]: failed to encode map e578 with expected crc 2024-03-29T05:02:17.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:17 smithi181 ceph-mon[101970]: failed to encode map e578 with expected crc 2024-03-29T05:02:17.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:17 smithi181 ceph-mon[101970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:02:17.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:17 smithi181 ceph-mon[101970]: osdmap e579: 8 total, 8 up, 8 in 2024-03-29T05:02:18.375 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:18 smithi187 ceph-mon[93536]: failed to encode map e579 with expected crc 2024-03-29T05:02:18.375 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:18 smithi187 ceph-mon[93536]: failed to encode map e579 with expected crc 2024-03-29T05:02:18.376 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:18 smithi187 ceph-mon[93536]: Health check update: Degraded data redundancy: 15/75 objects degraded (20.000%), 10 pgs degraded (PG_DEGRADED) 2024-03-29T05:02:18.376 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:18 smithi187 ceph-mon[93536]: pgmap v539: 137 pgs: 3 unknown, 1 active+undersized+degraded, 16 active+undersized, 117 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 175 KiB/s rd, 18 KiB/s wr, 239 op/s; 4/198 objects degraded (2.020%); 40 B/s, 15 keys/s, 2 objects/s recovering 2024-03-29T05:02:18.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:18 smithi181 ceph-mon[101970]: failed to encode map e579 with expected crc 2024-03-29T05:02:18.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:18 smithi181 ceph-mon[101970]: failed to encode map e579 with expected crc 2024-03-29T05:02:18.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:18 smithi181 ceph-mon[101970]: Health check update: Degraded data redundancy: 15/75 objects degraded (20.000%), 10 pgs degraded (PG_DEGRADED) 2024-03-29T05:02:18.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:18 smithi181 ceph-mon[101970]: pgmap v539: 137 pgs: 3 unknown, 1 active+undersized+degraded, 16 active+undersized, 117 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 175 KiB/s rd, 18 KiB/s wr, 239 op/s; 4/198 objects degraded (2.020%); 40 B/s, 15 keys/s, 2 objects/s recovering 2024-03-29T05:02:18.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:18 smithi181 ceph-mon[106466]: failed to encode map e579 with expected crc 2024-03-29T05:02:18.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:18 smithi181 ceph-mon[106466]: failed to encode map e579 with expected crc 2024-03-29T05:02:18.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:18 smithi181 ceph-mon[106466]: Health check update: Degraded data redundancy: 15/75 objects degraded (20.000%), 10 pgs degraded (PG_DEGRADED) 2024-03-29T05:02:18.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:18 smithi181 ceph-mon[106466]: pgmap v539: 137 pgs: 3 unknown, 1 active+undersized+degraded, 16 active+undersized, 117 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 175 KiB/s rd, 18 KiB/s wr, 239 op/s; 4/198 objects degraded (2.020%); 40 B/s, 15 keys/s, 2 objects/s recovering 2024-03-29T05:02:19.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:02:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:02:18] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.43.0" 2024-03-29T05:02:19.296 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:19.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: osdmap e580: 8 total, 8 up, 8 in 2024-03-29T05:02:19.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3911882274' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:19.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.639 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:19.639 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:19.639 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:19.639 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:02:19.639 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:02:19.639 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: pgmap v541: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 229 KiB/s rd, 25 KiB/s wr, 316 op/s; 89 B/s, 21 keys/s, 5 objects/s recovering 2024-03-29T05:02:19.639 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: pgmap v542: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 286 KiB/s rd, 32 KiB/s wr, 396 op/s; 111 B/s, 27 keys/s, 6 objects/s recovering 2024-03-29T05:02:19.639 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:19.639 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:02:19.639 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:02:19.639 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:02:19.639 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: pgmap v543: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 311 KiB/s rd, 35 KiB/s wr, 431 op/s; 121 B/s, 29 keys/s, 7 objects/s recovering 2024-03-29T05:02:19.639 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: pgmap v544: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 5.8 KiB/s wr, 46 op/s; 92 B/s, 5 keys/s, 4 objects/s recovering 2024-03-29T05:02:19.639 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:19 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:19.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: osdmap e580: 8 total, 8 up, 8 in 2024-03-29T05:02:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3911882274' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:02:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:02:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: pgmap v541: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 229 KiB/s rd, 25 KiB/s wr, 316 op/s; 89 B/s, 21 keys/s, 5 objects/s recovering 2024-03-29T05:02:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: pgmap v542: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 286 KiB/s rd, 32 KiB/s wr, 396 op/s; 111 B/s, 27 keys/s, 6 objects/s recovering 2024-03-29T05:02:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:02:19.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:02:19.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:02:19.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: pgmap v543: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 311 KiB/s rd, 35 KiB/s wr, 431 op/s; 121 B/s, 29 keys/s, 7 objects/s recovering 2024-03-29T05:02:19.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: pgmap v544: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 5.8 KiB/s wr, 46 op/s; 92 B/s, 5 keys/s, 4 objects/s recovering 2024-03-29T05:02:19.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:19.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: osdmap e580: 8 total, 8 up, 8 in 2024-03-29T05:02:19.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3911882274' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:19.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: failed to encode map e580 with expected crc 2024-03-29T05:02:19.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:19.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:19.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:19.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:02:19.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:02:19.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: pgmap v541: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 229 KiB/s rd, 25 KiB/s wr, 316 op/s; 89 B/s, 21 keys/s, 5 objects/s recovering 2024-03-29T05:02:19.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: pgmap v542: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 286 KiB/s rd, 32 KiB/s wr, 396 op/s; 111 B/s, 27 keys/s, 6 objects/s recovering 2024-03-29T05:02:19.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:19.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:02:19.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:02:19.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:02:19.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: pgmap v543: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 311 KiB/s rd, 35 KiB/s wr, 431 op/s; 121 B/s, 29 keys/s, 7 objects/s recovering 2024-03-29T05:02:19.709 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: pgmap v544: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 5.8 KiB/s wr, 46 op/s; 92 B/s, 5 keys/s, 4 objects/s recovering 2024-03-29T05:02:19.709 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:19 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:20.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:20 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:20.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:20 smithi187 ceph-mon[93536]: osdmap e581: 8 total, 8 up, 8 in 2024-03-29T05:02:20.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:20 smithi187 ceph-mon[93536]: failed to encode map e581 with expected crc 2024-03-29T05:02:20.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:20 smithi187 ceph-mon[93536]: failed to encode map e581 with expected crc 2024-03-29T05:02:20.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:20 smithi187 ceph-mon[93536]: failed to encode map e581 with expected crc 2024-03-29T05:02:20.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:20 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:02:20.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:20 smithi187 ceph-mon[93536]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/198 objects degraded (2.020%), 1 pg degraded) 2024-03-29T05:02:20.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:20 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:20.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:20 smithi181 ceph-mon[101970]: osdmap e581: 8 total, 8 up, 8 in 2024-03-29T05:02:20.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:20 smithi181 ceph-mon[101970]: failed to encode map e581 with expected crc 2024-03-29T05:02:20.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:20 smithi181 ceph-mon[101970]: failed to encode map e581 with expected crc 2024-03-29T05:02:20.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:20 smithi181 ceph-mon[101970]: failed to encode map e581 with expected crc 2024-03-29T05:02:20.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:20 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:02:20.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:20 smithi181 ceph-mon[101970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/198 objects degraded (2.020%), 1 pg degraded) 2024-03-29T05:02:20.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:20 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:20.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:20 smithi181 ceph-mon[106466]: osdmap e581: 8 total, 8 up, 8 in 2024-03-29T05:02:20.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:20 smithi181 ceph-mon[106466]: failed to encode map e581 with expected crc 2024-03-29T05:02:20.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:20 smithi181 ceph-mon[106466]: failed to encode map e581 with expected crc 2024-03-29T05:02:20.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:20 smithi181 ceph-mon[106466]: failed to encode map e581 with expected crc 2024-03-29T05:02:20.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:20 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:02:20.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:20 smithi181 ceph-mon[106466]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/198 objects degraded (2.020%), 1 pg degraded) 2024-03-29T05:02:21.315 INFO:tasks.workunit.client.0.smithi181.stdout:num images is: 1 2024-03-29T05:02:21.316 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 1 2024-03-29T05:02:21.316 INFO:tasks.workunit.client.0.smithi181.stdout:image: image20 2024-03-29T05:02:21.316 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image20 2024-03-29T05:02:21.316 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:21.326 INFO:tasks.workunit.client.0.smithi181.stdout:num images is: 2 2024-03-29T05:02:21.326 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 2 2024-03-29T05:02:21.326 INFO:tasks.workunit.client.0.smithi181.stdout:image: image20 2024-03-29T05:02:21.326 INFO:tasks.workunit.client.0.smithi181.stdout:image: image21 2024-03-29T05:02:21.326 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image20 2024-03-29T05:02:21.327 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image21 2024-03-29T05:02:21.381 INFO:tasks.workunit.client.0.smithi181.stdout:num images is: 1 2024-03-29T05:02:21.381 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 1 2024-03-29T05:02:21.381 INFO:tasks.workunit.client.0.smithi181.stdout:image: image21 2024-03-29T05:02:21.381 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image21 2024-03-29T05:02:21.381 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCreateLsDeletePP (4039 ms) 2024-03-29T05:02:21.381 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCopy 2024-03-29T05:02:21.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:21 smithi181 ceph-mon[106466]: failed to encode map e582 with expected crc 2024-03-29T05:02:21.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:21 smithi181 ceph-mon[106466]: osdmap e582: 8 total, 8 up, 8 in 2024-03-29T05:02:21.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:21 smithi181 ceph-mon[106466]: failed to encode map e582 with expected crc 2024-03-29T05:02:21.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:21 smithi181 ceph-mon[106466]: pgmap v547: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:02:21.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:21 smithi181 ceph-mon[101970]: failed to encode map e582 with expected crc 2024-03-29T05:02:21.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:21 smithi181 ceph-mon[101970]: osdmap e582: 8 total, 8 up, 8 in 2024-03-29T05:02:21.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:21 smithi181 ceph-mon[101970]: failed to encode map e582 with expected crc 2024-03-29T05:02:21.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:21 smithi181 ceph-mon[101970]: pgmap v547: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:02:21.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:21 smithi187 ceph-mon[93536]: failed to encode map e582 with expected crc 2024-03-29T05:02:21.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:21 smithi187 ceph-mon[93536]: osdmap e582: 8 total, 8 up, 8 in 2024-03-29T05:02:21.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:21 smithi187 ceph-mon[93536]: failed to encode map e582 with expected crc 2024-03-29T05:02:21.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:21 smithi187 ceph-mon[93536]: pgmap v547: 169 pgs: 32 unknown, 137 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:02:22.609 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:22.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:22 smithi187 ceph-mon[93536]: failed to encode map e582 with expected crc 2024-03-29T05:02:22.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:22 smithi187 ceph-mon[93536]: failed to encode map e582 with expected crc 2024-03-29T05:02:22.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:22 smithi187 ceph-mon[93536]: failed to encode map e582 with expected crc 2024-03-29T05:02:22.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:22 smithi187 ceph-mon[93536]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:02:22.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:22 smithi187 ceph-mon[93536]: failed to encode map e583 with expected crc 2024-03-29T05:02:22.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:22 smithi187 ceph-mon[93536]: osdmap e583: 8 total, 8 up, 8 in 2024-03-29T05:02:22.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:22 smithi187 ceph-mon[93536]: failed to encode map e583 with expected crc 2024-03-29T05:02:22.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:22 smithi187 ceph-mon[93536]: failed to encode map e583 with expected crc 2024-03-29T05:02:22.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:22 smithi181 ceph-mon[101970]: failed to encode map e582 with expected crc 2024-03-29T05:02:22.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:22 smithi181 ceph-mon[101970]: failed to encode map e582 with expected crc 2024-03-29T05:02:22.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:22 smithi181 ceph-mon[101970]: failed to encode map e582 with expected crc 2024-03-29T05:02:22.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:22 smithi181 ceph-mon[101970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:02:22.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:22 smithi181 ceph-mon[101970]: failed to encode map e583 with expected crc 2024-03-29T05:02:22.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:22 smithi181 ceph-mon[101970]: osdmap e583: 8 total, 8 up, 8 in 2024-03-29T05:02:22.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:22 smithi181 ceph-mon[101970]: failed to encode map e583 with expected crc 2024-03-29T05:02:22.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:22 smithi181 ceph-mon[101970]: failed to encode map e583 with expected crc 2024-03-29T05:02:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:22 smithi181 ceph-mon[106466]: failed to encode map e582 with expected crc 2024-03-29T05:02:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:22 smithi181 ceph-mon[106466]: failed to encode map e582 with expected crc 2024-03-29T05:02:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:22 smithi181 ceph-mon[106466]: failed to encode map e582 with expected crc 2024-03-29T05:02:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:22 smithi181 ceph-mon[106466]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:02:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:22 smithi181 ceph-mon[106466]: failed to encode map e583 with expected crc 2024-03-29T05:02:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:22 smithi181 ceph-mon[106466]: osdmap e583: 8 total, 8 up, 8 in 2024-03-29T05:02:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:22 smithi181 ceph-mon[106466]: failed to encode map e583 with expected crc 2024-03-29T05:02:22.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:22 smithi181 ceph-mon[106466]: failed to encode map e583 with expected crc 2024-03-29T05:02:23.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:23 smithi187 ceph-mon[93536]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:23 smithi187 ceph-mon[93536]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:23 smithi187 ceph-mon[93536]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:23 smithi187 ceph-mon[93536]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:23 smithi187 ceph-mon[93536]: osdmap e584: 8 total, 8 up, 8 in 2024-03-29T05:02:23.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:23 smithi187 ceph-mon[93536]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:23 smithi187 ceph-mon[93536]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:23 smithi187 ceph-mon[93536]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:23 smithi187 ceph-mon[93536]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:23 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2093036975' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:23.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:23 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:23.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:23 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:23.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:23 smithi187 ceph-mon[93536]: osdmap e585: 8 total, 8 up, 8 in 2024-03-29T05:02:23.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:23 smithi187 ceph-mon[93536]: failed to encode map e585 with expected crc 2024-03-29T05:02:23.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:23 smithi187 ceph-mon[93536]: pgmap v551: 201 pgs: 35 unknown, 166 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 6.6 KiB/s wr, 91 op/s 2024-03-29T05:02:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[106466]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[106466]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[106466]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[106466]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[106466]: osdmap e584: 8 total, 8 up, 8 in 2024-03-29T05:02:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[106466]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[106466]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[106466]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[106466]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2093036975' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:23.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[106466]: osdmap e585: 8 total, 8 up, 8 in 2024-03-29T05:02:23.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[106466]: failed to encode map e585 with expected crc 2024-03-29T05:02:23.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[106466]: pgmap v551: 201 pgs: 35 unknown, 166 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 6.6 KiB/s wr, 91 op/s 2024-03-29T05:02:23.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[101970]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[101970]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[101970]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[101970]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[101970]: osdmap e584: 8 total, 8 up, 8 in 2024-03-29T05:02:23.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[101970]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[101970]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[101970]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[101970]: failed to encode map e584 with expected crc 2024-03-29T05:02:23.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2093036975' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:23.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:23.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:23.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[101970]: osdmap e585: 8 total, 8 up, 8 in 2024-03-29T05:02:23.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[101970]: failed to encode map e585 with expected crc 2024-03-29T05:02:23.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:23 smithi181 ceph-mon[101970]: pgmap v551: 201 pgs: 35 unknown, 166 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 6.6 KiB/s wr, 91 op/s 2024-03-29T05:02:24.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:24 smithi187 ceph-mon[93536]: failed to encode map e585 with expected crc 2024-03-29T05:02:24.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:24 smithi187 ceph-mon[93536]: failed to encode map e585 with expected crc 2024-03-29T05:02:24.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:24 smithi187 ceph-mon[93536]: osdmap e586: 8 total, 8 up, 8 in 2024-03-29T05:02:24.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:24 smithi181 ceph-mon[101970]: failed to encode map e585 with expected crc 2024-03-29T05:02:24.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:24 smithi181 ceph-mon[101970]: failed to encode map e585 with expected crc 2024-03-29T05:02:24.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:24 smithi181 ceph-mon[101970]: osdmap e586: 8 total, 8 up, 8 in 2024-03-29T05:02:24.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:24 smithi181 ceph-mon[106466]: failed to encode map e585 with expected crc 2024-03-29T05:02:24.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:24 smithi181 ceph-mon[106466]: failed to encode map e585 with expected crc 2024-03-29T05:02:24.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:24 smithi181 ceph-mon[106466]: osdmap e586: 8 total, 8 up, 8 in 2024-03-29T05:02:24.988 INFO:tasks.workunit.client.0.smithi181.stdout:image: image22 2024-03-29T05:02:24.988 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image22 2024-03-29T05:02:24.988 INFO:tasks.workunit.client.0.smithi181.stdout:found image22 2024-03-29T05:02:24.988 INFO:tasks.workunit.client.0.smithi181.stdout:erased image22 2024-03-29T05:02:24.988 INFO:tasks.workunit.client.0.smithi181.stdout:image: image22 2024-03-29T05:02:24.989 INFO:tasks.workunit.client.0.smithi181.stdout:image: image23 2024-03-29T05:02:24.989 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image22 2024-03-29T05:02:24.989 INFO:tasks.workunit.client.0.smithi181.stdout:found image22 2024-03-29T05:02:24.989 INFO:tasks.workunit.client.0.smithi181.stdout:erased image22 2024-03-29T05:02:24.989 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image23 2024-03-29T05:02:24.989 INFO:tasks.workunit.client.0.smithi181.stdout:found image23 2024-03-29T05:02:24.989 INFO:tasks.workunit.client.0.smithi181.stdout:erased image23 2024-03-29T05:02:24.989 INFO:tasks.workunit.client.0.smithi181.stdout:100.00% done 2024-03-29T05:02:24.989 INFO:tasks.workunit.client.0.smithi181.stdout:image: image22 2024-03-29T05:02:24.989 INFO:tasks.workunit.client.0.smithi181.stdout:image: image23 2024-03-29T05:02:24.989 INFO:tasks.workunit.client.0.smithi181.stdout:image: image24 2024-03-29T05:02:24.989 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image22 2024-03-29T05:02:24.989 INFO:tasks.workunit.client.0.smithi181.stdout:found image22 2024-03-29T05:02:24.989 INFO:tasks.workunit.client.0.smithi181.stdout:erased image22 2024-03-29T05:02:24.989 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image23 2024-03-29T05:02:24.990 INFO:tasks.workunit.client.0.smithi181.stdout:found image23 2024-03-29T05:02:24.990 INFO:tasks.workunit.client.0.smithi181.stdout:erased image23 2024-03-29T05:02:24.990 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image24 2024-03-29T05:02:24.990 INFO:tasks.workunit.client.0.smithi181.stdout:found image24 2024-03-29T05:02:24.990 INFO:tasks.workunit.client.0.smithi181.stdout:erased image24 2024-03-29T05:02:24.990 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCopy (3607 ms) 2024-03-29T05:02:24.990 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCopyPP 2024-03-29T05:02:25.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:25 smithi187 ceph-mon[93536]: failed to encode map e586 with expected crc 2024-03-29T05:02:25.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:25 smithi187 ceph-mon[93536]: failed to encode map e586 with expected crc 2024-03-29T05:02:25.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:25 smithi187 ceph-mon[93536]: failed to encode map e586 with expected crc 2024-03-29T05:02:25.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:25 smithi187 ceph-mon[93536]: failed to encode map e586 with expected crc 2024-03-29T05:02:25.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:25 smithi187 ceph-mon[93536]: osdmap e587: 8 total, 8 up, 8 in 2024-03-29T05:02:25.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:25 smithi187 ceph-mon[93536]: pgmap v554: 201 pgs: 26 unknown, 175 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 6.9 KiB/s wr, 93 op/s 2024-03-29T05:02:25.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:25 smithi187 ceph-mon[93536]: failed to encode map e587 with expected crc 2024-03-29T05:02:25.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:25 smithi181 ceph-mon[101970]: failed to encode map e586 with expected crc 2024-03-29T05:02:25.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:25 smithi181 ceph-mon[101970]: failed to encode map e586 with expected crc 2024-03-29T05:02:25.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:25 smithi181 ceph-mon[101970]: failed to encode map e586 with expected crc 2024-03-29T05:02:25.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:25 smithi181 ceph-mon[101970]: failed to encode map e586 with expected crc 2024-03-29T05:02:25.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:25 smithi181 ceph-mon[101970]: osdmap e587: 8 total, 8 up, 8 in 2024-03-29T05:02:25.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:25 smithi181 ceph-mon[101970]: pgmap v554: 201 pgs: 26 unknown, 175 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 6.9 KiB/s wr, 93 op/s 2024-03-29T05:02:25.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:25 smithi181 ceph-mon[101970]: failed to encode map e587 with expected crc 2024-03-29T05:02:25.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:25 smithi181 ceph-mon[106466]: failed to encode map e586 with expected crc 2024-03-29T05:02:25.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:25 smithi181 ceph-mon[106466]: failed to encode map e586 with expected crc 2024-03-29T05:02:25.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:25 smithi181 ceph-mon[106466]: failed to encode map e586 with expected crc 2024-03-29T05:02:25.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:25 smithi181 ceph-mon[106466]: failed to encode map e586 with expected crc 2024-03-29T05:02:25.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:25 smithi181 ceph-mon[106466]: osdmap e587: 8 total, 8 up, 8 in 2024-03-29T05:02:25.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:25 smithi181 ceph-mon[106466]: pgmap v554: 201 pgs: 26 unknown, 175 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 6.9 KiB/s wr, 93 op/s 2024-03-29T05:02:25.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:25 smithi181 ceph-mon[106466]: failed to encode map e587 with expected crc 2024-03-29T05:02:26.623 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:26.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:26 smithi187 ceph-mon[93536]: failed to encode map e587 with expected crc 2024-03-29T05:02:26.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:26 smithi187 ceph-mon[93536]: failed to encode map e588 with expected crc 2024-03-29T05:02:26.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:26 smithi187 ceph-mon[93536]: osdmap e588: 8 total, 8 up, 8 in 2024-03-29T05:02:26.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:26 smithi187 ceph-mon[93536]: failed to encode map e588 with expected crc 2024-03-29T05:02:26.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:26 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/151544364' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:26.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:26 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:26.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:26.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:02:26.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:26 smithi181 ceph-mon[101970]: failed to encode map e587 with expected crc 2024-03-29T05:02:26.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:26 smithi181 ceph-mon[101970]: failed to encode map e588 with expected crc 2024-03-29T05:02:26.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:26 smithi181 ceph-mon[101970]: osdmap e588: 8 total, 8 up, 8 in 2024-03-29T05:02:26.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:26 smithi181 ceph-mon[101970]: failed to encode map e588 with expected crc 2024-03-29T05:02:26.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:26 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/151544364' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:26.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:26 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:26.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:26.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:02:26.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:26 smithi181 ceph-mon[106466]: failed to encode map e587 with expected crc 2024-03-29T05:02:26.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:26 smithi181 ceph-mon[106466]: failed to encode map e588 with expected crc 2024-03-29T05:02:26.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:26 smithi181 ceph-mon[106466]: osdmap e588: 8 total, 8 up, 8 in 2024-03-29T05:02:26.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:26 smithi181 ceph-mon[106466]: failed to encode map e588 with expected crc 2024-03-29T05:02:26.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:26 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/151544364' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:26.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:26 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:26.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:26.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:02:27.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:27 smithi187 ceph-mon[93536]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:27 smithi187 ceph-mon[93536]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:27 smithi187 ceph-mon[93536]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:27 smithi187 ceph-mon[93536]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:27 smithi187 ceph-mon[93536]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:27 smithi187 ceph-mon[93536]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:27 smithi187 ceph-mon[93536]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:27 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:27.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:27 smithi187 ceph-mon[93536]: pgmap v557: 233 pgs: 16 creating+peering, 31 unknown, 186 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 82 KiB/s rd, 3.7 KiB/s wr, 91 op/s 2024-03-29T05:02:27.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:27 smithi187 ceph-mon[93536]: osdmap e589: 8 total, 8 up, 8 in 2024-03-29T05:02:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[101970]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[101970]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[101970]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[101970]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[101970]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[101970]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[101970]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:27.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[101970]: pgmap v557: 233 pgs: 16 creating+peering, 31 unknown, 186 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 82 KiB/s rd, 3.7 KiB/s wr, 91 op/s 2024-03-29T05:02:27.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[101970]: osdmap e589: 8 total, 8 up, 8 in 2024-03-29T05:02:27.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[106466]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[106466]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[106466]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[106466]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[106466]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[106466]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[106466]: failed to encode map e588 with expected crc 2024-03-29T05:02:27.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:27.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[106466]: pgmap v557: 233 pgs: 16 creating+peering, 31 unknown, 186 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 82 KiB/s rd, 3.7 KiB/s wr, 91 op/s 2024-03-29T05:02:27.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:27 smithi181 ceph-mon[106466]: osdmap e589: 8 total, 8 up, 8 in 2024-03-29T05:02:28.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:28 smithi187 ceph-mon[93536]: failed to encode map e589 with expected crc 2024-03-29T05:02:28.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:28 smithi187 ceph-mon[93536]: failed to encode map e589 with expected crc 2024-03-29T05:02:28.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:28 smithi187 ceph-mon[93536]: failed to encode map e589 with expected crc 2024-03-29T05:02:28.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:28 smithi187 ceph-mon[93536]: failed to encode map e589 with expected crc 2024-03-29T05:02:28.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:28 smithi187 ceph-mon[93536]: osdmap e590: 8 total, 8 up, 8 in 2024-03-29T05:02:28.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:28 smithi181 ceph-mon[106466]: failed to encode map e589 with expected crc 2024-03-29T05:02:28.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:28 smithi181 ceph-mon[106466]: failed to encode map e589 with expected crc 2024-03-29T05:02:28.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:28 smithi181 ceph-mon[106466]: failed to encode map e589 with expected crc 2024-03-29T05:02:28.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:28 smithi181 ceph-mon[106466]: failed to encode map e589 with expected crc 2024-03-29T05:02:28.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:28 smithi181 ceph-mon[106466]: osdmap e590: 8 total, 8 up, 8 in 2024-03-29T05:02:28.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:28 smithi181 ceph-mon[101970]: failed to encode map e589 with expected crc 2024-03-29T05:02:28.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:28 smithi181 ceph-mon[101970]: failed to encode map e589 with expected crc 2024-03-29T05:02:28.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:28 smithi181 ceph-mon[101970]: failed to encode map e589 with expected crc 2024-03-29T05:02:28.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:28 smithi181 ceph-mon[101970]: failed to encode map e589 with expected crc 2024-03-29T05:02:28.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:28 smithi181 ceph-mon[101970]: osdmap e590: 8 total, 8 up, 8 in 2024-03-29T05:02:28.881 INFO:tasks.workunit.client.0.smithi181.stdout:num images is: 1 2024-03-29T05:02:28.882 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 1 2024-03-29T05:02:28.882 INFO:tasks.workunit.client.0.smithi181.stdout:image: image25 2024-03-29T05:02:28.882 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image25 2024-03-29T05:02:28.899 INFO:tasks.workunit.client.0.smithi181.stdout:num images is: 2 2024-03-29T05:02:28.899 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 2 2024-03-29T05:02:28.899 INFO:tasks.workunit.client.0.smithi181.stdout:image: image25 2024-03-29T05:02:28.900 INFO:tasks.workunit.client.0.smithi181.stdout:image: image26 2024-03-29T05:02:28.900 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image25 2024-03-29T05:02:28.900 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image26 2024-03-29T05:02:28.941 INFO:tasks.workunit.client.0.smithi181.stdout:100.00% done 2024-03-29T05:02:28.941 INFO:tasks.workunit.client.0.smithi181.stdout:num images is: 3 2024-03-29T05:02:28.941 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 3 2024-03-29T05:02:28.942 INFO:tasks.workunit.client.0.smithi181.stdout:image: image25 2024-03-29T05:02:28.942 INFO:tasks.workunit.client.0.smithi181.stdout:image: image26 2024-03-29T05:02:28.942 INFO:tasks.workunit.client.0.smithi181.stdout:image: image27 2024-03-29T05:02:28.942 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image25 2024-03-29T05:02:28.942 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image26 2024-03-29T05:02:28.942 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image27 2024-03-29T05:02:28.970 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCopyPP (3982 ms) 2024-03-29T05:02:28.970 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestDeepCopy 2024-03-29T05:02:29.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:02:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:02:28] "GET /metrics HTTP/1.1" 200 45980 "" "Prometheus/2.43.0" 2024-03-29T05:02:29.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:29 smithi187 ceph-mon[93536]: failed to encode map e590 with expected crc 2024-03-29T05:02:29.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:29 smithi187 ceph-mon[93536]: failed to encode map e590 with expected crc 2024-03-29T05:02:29.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:29 smithi187 ceph-mon[93536]: pgmap v559: 233 pgs: 5 creating+activating, 21 creating+peering, 207 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 398 op/s 2024-03-29T05:02:29.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:29 smithi187 ceph-mon[93536]: osdmap e591: 8 total, 8 up, 8 in 2024-03-29T05:02:29.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:29 smithi187 ceph-mon[93536]: failed to encode map e591 with expected crc 2024-03-29T05:02:29.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:29 smithi181 ceph-mon[101970]: failed to encode map e590 with expected crc 2024-03-29T05:02:29.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:29 smithi181 ceph-mon[101970]: failed to encode map e590 with expected crc 2024-03-29T05:02:29.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:29 smithi181 ceph-mon[101970]: pgmap v559: 233 pgs: 5 creating+activating, 21 creating+peering, 207 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 398 op/s 2024-03-29T05:02:29.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:29 smithi181 ceph-mon[101970]: osdmap e591: 8 total, 8 up, 8 in 2024-03-29T05:02:29.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:29 smithi181 ceph-mon[101970]: failed to encode map e591 with expected crc 2024-03-29T05:02:29.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:29 smithi181 ceph-mon[106466]: failed to encode map e590 with expected crc 2024-03-29T05:02:29.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:29 smithi181 ceph-mon[106466]: failed to encode map e590 with expected crc 2024-03-29T05:02:29.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:29 smithi181 ceph-mon[106466]: pgmap v559: 233 pgs: 5 creating+activating, 21 creating+peering, 207 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 398 op/s 2024-03-29T05:02:29.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:29 smithi181 ceph-mon[106466]: osdmap e591: 8 total, 8 up, 8 in 2024-03-29T05:02:29.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:29 smithi181 ceph-mon[106466]: failed to encode map e591 with expected crc 2024-03-29T05:02:30.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:30 smithi187 ceph-mon[93536]: failed to encode map e591 with expected crc 2024-03-29T05:02:30.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:30 smithi187 ceph-mon[93536]: failed to encode map e591 with expected crc 2024-03-29T05:02:30.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:30 smithi187 ceph-mon[93536]: osdmap e592: 8 total, 8 up, 8 in 2024-03-29T05:02:30.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:30 smithi187 ceph-mon[93536]: failed to encode map e592 with expected crc 2024-03-29T05:02:30.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:30 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1221100732' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:30.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:30 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:30.637 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:30.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:30 smithi181 ceph-mon[106466]: failed to encode map e591 with expected crc 2024-03-29T05:02:30.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:30 smithi181 ceph-mon[106466]: failed to encode map e591 with expected crc 2024-03-29T05:02:30.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:30 smithi181 ceph-mon[106466]: osdmap e592: 8 total, 8 up, 8 in 2024-03-29T05:02:30.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:30 smithi181 ceph-mon[106466]: failed to encode map e592 with expected crc 2024-03-29T05:02:30.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:30 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1221100732' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:30.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:30 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:30.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:30 smithi181 ceph-mon[101970]: failed to encode map e591 with expected crc 2024-03-29T05:02:30.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:30 smithi181 ceph-mon[101970]: failed to encode map e591 with expected crc 2024-03-29T05:02:30.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:30 smithi181 ceph-mon[101970]: osdmap e592: 8 total, 8 up, 8 in 2024-03-29T05:02:30.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:30 smithi181 ceph-mon[101970]: failed to encode map e592 with expected crc 2024-03-29T05:02:30.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:30 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1221100732' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:30.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:30 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:31.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:31 smithi187 ceph-mon[93536]: failed to encode map e592 with expected crc 2024-03-29T05:02:31.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:31 smithi187 ceph-mon[93536]: pgmap v562: 265 pgs: 32 unknown, 5 creating+activating, 21 creating+peering, 207 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 259 KiB/s rd, 18 KiB/s wr, 307 op/s 2024-03-29T05:02:31.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:31 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:31.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:31 smithi187 ceph-mon[93536]: osdmap e593: 8 total, 8 up, 8 in 2024-03-29T05:02:31.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:31 smithi187 ceph-mon[93536]: failed to encode map e593 with expected crc 2024-03-29T05:02:31.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:31 smithi181 ceph-mon[101970]: failed to encode map e592 with expected crc 2024-03-29T05:02:31.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:31 smithi181 ceph-mon[101970]: pgmap v562: 265 pgs: 32 unknown, 5 creating+activating, 21 creating+peering, 207 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 259 KiB/s rd, 18 KiB/s wr, 307 op/s 2024-03-29T05:02:31.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:31 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:31.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:31 smithi181 ceph-mon[101970]: osdmap e593: 8 total, 8 up, 8 in 2024-03-29T05:02:31.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:31 smithi181 ceph-mon[101970]: failed to encode map e593 with expected crc 2024-03-29T05:02:31.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:31 smithi181 ceph-mon[106466]: failed to encode map e592 with expected crc 2024-03-29T05:02:31.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:31 smithi181 ceph-mon[106466]: pgmap v562: 265 pgs: 32 unknown, 5 creating+activating, 21 creating+peering, 207 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 259 KiB/s rd, 18 KiB/s wr, 307 op/s 2024-03-29T05:02:31.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:31 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:31.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:31 smithi181 ceph-mon[106466]: osdmap e593: 8 total, 8 up, 8 in 2024-03-29T05:02:31.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:31 smithi181 ceph-mon[106466]: failed to encode map e593 with expected crc 2024-03-29T05:02:32.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:32 smithi187 ceph-mon[93536]: failed to encode map e593 with expected crc 2024-03-29T05:02:32.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:32 smithi187 ceph-mon[93536]: failed to encode map e593 with expected crc 2024-03-29T05:02:32.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:32 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:02:32.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:32 smithi187 ceph-mon[93536]: osdmap e594: 8 total, 8 up, 8 in 2024-03-29T05:02:32.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:32 smithi181 ceph-mon[101970]: failed to encode map e593 with expected crc 2024-03-29T05:02:32.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:32 smithi181 ceph-mon[101970]: failed to encode map e593 with expected crc 2024-03-29T05:02:32.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:32 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:02:32.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:32 smithi181 ceph-mon[101970]: osdmap e594: 8 total, 8 up, 8 in 2024-03-29T05:02:32.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:32 smithi181 ceph-mon[106466]: failed to encode map e593 with expected crc 2024-03-29T05:02:32.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:32 smithi181 ceph-mon[106466]: failed to encode map e593 with expected crc 2024-03-29T05:02:32.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:32 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:02:32.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:32 smithi181 ceph-mon[106466]: osdmap e594: 8 total, 8 up, 8 in 2024-03-29T05:02:33.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:33 smithi187 ceph-mon[93536]: failed to encode map e594 with expected crc 2024-03-29T05:02:33.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:33 smithi187 ceph-mon[93536]: failed to encode map e594 with expected crc 2024-03-29T05:02:33.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:33 smithi187 ceph-mon[93536]: failed to encode map e594 with expected crc 2024-03-29T05:02:33.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:33 smithi187 ceph-mon[93536]: failed to encode map e594 with expected crc 2024-03-29T05:02:33.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:33 smithi187 ceph-mon[93536]: failed to encode map e594 with expected crc 2024-03-29T05:02:33.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:33 smithi187 ceph-mon[93536]: osdmap e595: 8 total, 8 up, 8 in 2024-03-29T05:02:33.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:33 smithi187 ceph-mon[93536]: failed to encode map e595 with expected crc 2024-03-29T05:02:33.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:33 smithi187 ceph-mon[93536]: pgmap v566: 265 pgs: 2 unknown, 263 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 396 op/s 2024-03-29T05:02:33.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:33 smithi181 ceph-mon[106466]: failed to encode map e594 with expected crc 2024-03-29T05:02:33.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:33 smithi181 ceph-mon[106466]: failed to encode map e594 with expected crc 2024-03-29T05:02:33.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:33 smithi181 ceph-mon[106466]: failed to encode map e594 with expected crc 2024-03-29T05:02:33.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:33 smithi181 ceph-mon[106466]: failed to encode map e594 with expected crc 2024-03-29T05:02:33.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:33 smithi181 ceph-mon[106466]: failed to encode map e594 with expected crc 2024-03-29T05:02:33.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:33 smithi181 ceph-mon[106466]: osdmap e595: 8 total, 8 up, 8 in 2024-03-29T05:02:33.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:33 smithi181 ceph-mon[106466]: failed to encode map e595 with expected crc 2024-03-29T05:02:33.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:33 smithi181 ceph-mon[106466]: pgmap v566: 265 pgs: 2 unknown, 263 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 396 op/s 2024-03-29T05:02:33.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:33 smithi181 ceph-mon[101970]: failed to encode map e594 with expected crc 2024-03-29T05:02:33.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:33 smithi181 ceph-mon[101970]: failed to encode map e594 with expected crc 2024-03-29T05:02:33.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:33 smithi181 ceph-mon[101970]: failed to encode map e594 with expected crc 2024-03-29T05:02:33.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:33 smithi181 ceph-mon[101970]: failed to encode map e594 with expected crc 2024-03-29T05:02:33.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:33 smithi181 ceph-mon[101970]: failed to encode map e594 with expected crc 2024-03-29T05:02:33.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:33 smithi181 ceph-mon[101970]: osdmap e595: 8 total, 8 up, 8 in 2024-03-29T05:02:33.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:33 smithi181 ceph-mon[101970]: failed to encode map e595 with expected crc 2024-03-29T05:02:33.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:33 smithi181 ceph-mon[101970]: pgmap v566: 265 pgs: 2 unknown, 263 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 396 op/s 2024-03-29T05:02:34.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:34 smithi187 ceph-mon[93536]: failed to encode map e595 with expected crc 2024-03-29T05:02:34.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:34 smithi187 ceph-mon[93536]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:02:34.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:34 smithi187 ceph-mon[93536]: osdmap e596: 8 total, 8 up, 8 in 2024-03-29T05:02:34.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:34 smithi187 ceph-mon[93536]: failed to encode map e596 with expected crc 2024-03-29T05:02:34.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:34 smithi181 ceph-mon[101970]: failed to encode map e595 with expected crc 2024-03-29T05:02:34.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:34 smithi181 ceph-mon[101970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:02:34.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:34 smithi181 ceph-mon[101970]: osdmap e596: 8 total, 8 up, 8 in 2024-03-29T05:02:34.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:34 smithi181 ceph-mon[101970]: failed to encode map e596 with expected crc 2024-03-29T05:02:34.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:34 smithi181 ceph-mon[106466]: failed to encode map e595 with expected crc 2024-03-29T05:02:34.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:34 smithi181 ceph-mon[106466]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:02:34.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:34 smithi181 ceph-mon[106466]: osdmap e596: 8 total, 8 up, 8 in 2024-03-29T05:02:34.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:34 smithi181 ceph-mon[106466]: failed to encode map e596 with expected crc 2024-03-29T05:02:35.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:35 smithi187 ceph-mon[93536]: failed to encode map e596 with expected crc 2024-03-29T05:02:35.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:35 smithi187 ceph-mon[93536]: failed to encode map e596 with expected crc 2024-03-29T05:02:35.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:35 smithi187 ceph-mon[93536]: osdmap e597: 8 total, 8 up, 8 in 2024-03-29T05:02:35.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:35 smithi187 ceph-mon[93536]: pgmap v569: 265 pgs: 265 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 342 KiB/s rd, 22 KiB/s wr, 399 op/s 2024-03-29T05:02:35.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:35 smithi187 ceph-mon[93536]: failed to encode map e597 with expected crc 2024-03-29T05:02:35.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:35 smithi181 ceph-mon[101970]: failed to encode map e596 with expected crc 2024-03-29T05:02:35.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:35 smithi181 ceph-mon[101970]: failed to encode map e596 with expected crc 2024-03-29T05:02:35.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:35 smithi181 ceph-mon[101970]: osdmap e597: 8 total, 8 up, 8 in 2024-03-29T05:02:35.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:35 smithi181 ceph-mon[101970]: pgmap v569: 265 pgs: 265 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 342 KiB/s rd, 22 KiB/s wr, 399 op/s 2024-03-29T05:02:35.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:35 smithi181 ceph-mon[101970]: failed to encode map e597 with expected crc 2024-03-29T05:02:35.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:35 smithi181 ceph-mon[106466]: failed to encode map e596 with expected crc 2024-03-29T05:02:35.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:35 smithi181 ceph-mon[106466]: failed to encode map e596 with expected crc 2024-03-29T05:02:35.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:35 smithi181 ceph-mon[106466]: osdmap e597: 8 total, 8 up, 8 in 2024-03-29T05:02:35.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:35 smithi181 ceph-mon[106466]: pgmap v569: 265 pgs: 265 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 342 KiB/s rd, 22 KiB/s wr, 399 op/s 2024-03-29T05:02:35.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:35 smithi181 ceph-mon[106466]: failed to encode map e597 with expected crc 2024-03-29T05:02:36.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:36 smithi187 ceph-mon[93536]: failed to encode map e597 with expected crc 2024-03-29T05:02:36.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:36 smithi187 ceph-mon[93536]: failed to encode map e598 with expected crc 2024-03-29T05:02:36.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:36 smithi187 ceph-mon[93536]: osdmap e598: 8 total, 8 up, 8 in 2024-03-29T05:02:36.694 INFO:tasks.workunit.client.0.smithi181.stdout:image: image28 2024-03-29T05:02:36.694 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image28 2024-03-29T05:02:36.694 INFO:tasks.workunit.client.0.smithi181.stdout:found image28 2024-03-29T05:02:36.694 INFO:tasks.workunit.client.0.smithi181.stdout:erased image28 2024-03-29T05:02:36.694 INFO:tasks.workunit.client.0.smithi181.stdout:image: image28 2024-03-29T05:02:36.694 INFO:tasks.workunit.client.0.smithi181.stdout:image: image29 2024-03-29T05:02:36.694 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image28 2024-03-29T05:02:36.694 INFO:tasks.workunit.client.0.smithi181.stdout:found image28 2024-03-29T05:02:36.695 INFO:tasks.workunit.client.0.smithi181.stdout:erased image28 2024-03-29T05:02:36.695 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image29 2024-03-29T05:02:36.695 INFO:tasks.workunit.client.0.smithi181.stdout:found image29 2024-03-29T05:02:36.695 INFO:tasks.workunit.client.0.smithi181.stdout:erased image29 2024-03-29T05:02:36.695 INFO:tasks.workunit.client.0.smithi181.stdout:100.00% done 2024-03-29T05:02:36.695 INFO:tasks.workunit.client.0.smithi181.stdout:image: image28 2024-03-29T05:02:36.695 INFO:tasks.workunit.client.0.smithi181.stdout:image: image29 2024-03-29T05:02:36.695 INFO:tasks.workunit.client.0.smithi181.stdout:image: image30 2024-03-29T05:02:36.695 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image28 2024-03-29T05:02:36.695 INFO:tasks.workunit.client.0.smithi181.stdout:found image28 2024-03-29T05:02:36.695 INFO:tasks.workunit.client.0.smithi181.stdout:erased image28 2024-03-29T05:02:36.695 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image29 2024-03-29T05:02:36.695 INFO:tasks.workunit.client.0.smithi181.stdout:found image29 2024-03-29T05:02:36.695 INFO:tasks.workunit.client.0.smithi181.stdout:erased image29 2024-03-29T05:02:36.695 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image30 2024-03-29T05:02:36.695 INFO:tasks.workunit.client.0.smithi181.stdout:found image30 2024-03-29T05:02:36.696 INFO:tasks.workunit.client.0.smithi181.stdout:erased image30 2024-03-29T05:02:36.696 INFO:tasks.workunit.client.0.smithi181.stdout:image: image28 2024-03-29T05:02:36.696 INFO:tasks.workunit.client.0.smithi181.stdout:image: image29 2024-03-29T05:02:36.696 INFO:tasks.workunit.client.0.smithi181.stdout:image: image30 2024-03-29T05:02:36.696 INFO:tasks.workunit.client.0.smithi181.stdout:image: image31 2024-03-29T05:02:36.696 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image28 2024-03-29T05:02:36.696 INFO:tasks.workunit.client.0.smithi181.stdout:found image28 2024-03-29T05:02:36.696 INFO:tasks.workunit.client.0.smithi181.stdout:erased image28 2024-03-29T05:02:36.696 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image29 2024-03-29T05:02:36.696 INFO:tasks.workunit.client.0.smithi181.stdout:found image29 2024-03-29T05:02:36.696 INFO:tasks.workunit.client.0.smithi181.stdout:erased image29 2024-03-29T05:02:36.696 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image30 2024-03-29T05:02:36.696 INFO:tasks.workunit.client.0.smithi181.stdout:found image30 2024-03-29T05:02:36.696 INFO:tasks.workunit.client.0.smithi181.stdout:erased image30 2024-03-29T05:02:36.696 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image31 2024-03-29T05:02:36.697 INFO:tasks.workunit.client.0.smithi181.stdout:found image31 2024-03-29T05:02:36.697 INFO:tasks.workunit.client.0.smithi181.stdout:erased image31 2024-03-29T05:02:36.697 INFO:tasks.workunit.client.0.smithi181.stdout:image: image28 2024-03-29T05:02:36.697 INFO:tasks.workunit.client.0.smithi181.stdout:image: image29 2024-03-29T05:02:36.697 INFO:tasks.workunit.client.0.smithi181.stdout:image: image30 2024-03-29T05:02:36.697 INFO:tasks.workunit.client.0.smithi181.stdout:image: image31 2024-03-29T05:02:36.697 INFO:tasks.workunit.client.0.smithi181.stdout:image: image32 2024-03-29T05:02:36.697 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image28 2024-03-29T05:02:36.697 INFO:tasks.workunit.client.0.smithi181.stdout:found image28 2024-03-29T05:02:36.697 INFO:tasks.workunit.client.0.smithi181.stdout:erased image28 2024-03-29T05:02:36.697 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image29 2024-03-29T05:02:36.697 INFO:tasks.workunit.client.0.smithi181.stdout:found image29 2024-03-29T05:02:36.697 INFO:tasks.workunit.client.0.smithi181.stdout:erased image29 2024-03-29T05:02:36.697 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image30 2024-03-29T05:02:36.697 INFO:tasks.workunit.client.0.smithi181.stdout:found image30 2024-03-29T05:02:36.698 INFO:tasks.workunit.client.0.smithi181.stdout:erased image30 2024-03-29T05:02:36.698 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image31 2024-03-29T05:02:36.698 INFO:tasks.workunit.client.0.smithi181.stdout:found image31 2024-03-29T05:02:36.698 INFO:tasks.workunit.client.0.smithi181.stdout:erased image31 2024-03-29T05:02:36.698 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image32 2024-03-29T05:02:36.698 INFO:tasks.workunit.client.0.smithi181.stdout:found image32 2024-03-29T05:02:36.698 INFO:tasks.workunit.client.0.smithi181.stdout:erased image32 2024-03-29T05:02:36.698 INFO:tasks.workunit.client.0.smithi181.stdout:100.00% done 2024-03-29T05:02:36.698 INFO:tasks.workunit.client.0.smithi181.stdout:image: image28 2024-03-29T05:02:36.698 INFO:tasks.workunit.client.0.smithi181.stdout:image: image29 2024-03-29T05:02:36.698 INFO:tasks.workunit.client.0.smithi181.stdout:image: image30 2024-03-29T05:02:36.698 INFO:tasks.workunit.client.0.smithi181.stdout:image: image31 2024-03-29T05:02:36.698 INFO:tasks.workunit.client.0.smithi181.stdout:image: image32 2024-03-29T05:02:36.698 INFO:tasks.workunit.client.0.smithi181.stdout:image: image33 2024-03-29T05:02:36.698 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image28 2024-03-29T05:02:36.699 INFO:tasks.workunit.client.0.smithi181.stdout:found image28 2024-03-29T05:02:36.699 INFO:tasks.workunit.client.0.smithi181.stdout:erased image28 2024-03-29T05:02:36.699 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image29 2024-03-29T05:02:36.699 INFO:tasks.workunit.client.0.smithi181.stdout:found image29 2024-03-29T05:02:36.699 INFO:tasks.workunit.client.0.smithi181.stdout:erased image29 2024-03-29T05:02:36.699 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image30 2024-03-29T05:02:36.699 INFO:tasks.workunit.client.0.smithi181.stdout:found image30 2024-03-29T05:02:36.699 INFO:tasks.workunit.client.0.smithi181.stdout:erased image30 2024-03-29T05:02:36.699 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image31 2024-03-29T05:02:36.699 INFO:tasks.workunit.client.0.smithi181.stdout:found image31 2024-03-29T05:02:36.699 INFO:tasks.workunit.client.0.smithi181.stdout:erased image31 2024-03-29T05:02:36.699 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image32 2024-03-29T05:02:36.699 INFO:tasks.workunit.client.0.smithi181.stdout:found image32 2024-03-29T05:02:36.699 INFO:tasks.workunit.client.0.smithi181.stdout:erased image32 2024-03-29T05:02:36.699 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image33 2024-03-29T05:02:36.700 INFO:tasks.workunit.client.0.smithi181.stdout:found image33 2024-03-29T05:02:36.700 INFO:tasks.workunit.client.0.smithi181.stdout:erased image33 2024-03-29T05:02:36.700 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestDeepCopy (7724 ms) 2024-03-29T05:02:36.700 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestDeepCopyPP 2024-03-29T05:02:36.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:36 smithi181 ceph-mon[101970]: failed to encode map e597 with expected crc 2024-03-29T05:02:36.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:36 smithi181 ceph-mon[101970]: failed to encode map e598 with expected crc 2024-03-29T05:02:36.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:36 smithi181 ceph-mon[101970]: osdmap e598: 8 total, 8 up, 8 in 2024-03-29T05:02:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:36 smithi181 ceph-mon[106466]: failed to encode map e597 with expected crc 2024-03-29T05:02:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:36 smithi181 ceph-mon[106466]: failed to encode map e598 with expected crc 2024-03-29T05:02:36.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:36 smithi181 ceph-mon[106466]: osdmap e598: 8 total, 8 up, 8 in 2024-03-29T05:02:37.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:37 smithi187 ceph-mon[93536]: failed to encode map e598 with expected crc 2024-03-29T05:02:37.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:37 smithi187 ceph-mon[93536]: failed to encode map e598 with expected crc 2024-03-29T05:02:37.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:37 smithi187 ceph-mon[93536]: failed to encode map e598 with expected crc 2024-03-29T05:02:37.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:37 smithi187 ceph-mon[93536]: failed to encode map e598 with expected crc 2024-03-29T05:02:37.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:37 smithi187 ceph-mon[93536]: osdmap e599: 8 total, 8 up, 8 in 2024-03-29T05:02:37.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:37 smithi187 ceph-mon[93536]: pgmap v572: 265 pgs: 265 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 410 KiB/s rd, 25 KiB/s wr, 486 op/s 2024-03-29T05:02:37.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:37 smithi181 ceph-mon[101970]: failed to encode map e598 with expected crc 2024-03-29T05:02:37.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:37 smithi181 ceph-mon[101970]: failed to encode map e598 with expected crc 2024-03-29T05:02:37.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:37 smithi181 ceph-mon[101970]: failed to encode map e598 with expected crc 2024-03-29T05:02:37.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:37 smithi181 ceph-mon[101970]: failed to encode map e598 with expected crc 2024-03-29T05:02:37.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:37 smithi181 ceph-mon[101970]: osdmap e599: 8 total, 8 up, 8 in 2024-03-29T05:02:37.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:37 smithi181 ceph-mon[101970]: pgmap v572: 265 pgs: 265 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 410 KiB/s rd, 25 KiB/s wr, 486 op/s 2024-03-29T05:02:37.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:37 smithi181 ceph-mon[106466]: failed to encode map e598 with expected crc 2024-03-29T05:02:37.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:37 smithi181 ceph-mon[106466]: failed to encode map e598 with expected crc 2024-03-29T05:02:37.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:37 smithi181 ceph-mon[106466]: failed to encode map e598 with expected crc 2024-03-29T05:02:37.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:37 smithi181 ceph-mon[106466]: failed to encode map e598 with expected crc 2024-03-29T05:02:37.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:37 smithi181 ceph-mon[106466]: osdmap e599: 8 total, 8 up, 8 in 2024-03-29T05:02:37.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:37 smithi181 ceph-mon[106466]: pgmap v572: 265 pgs: 265 active+clean; 583 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 410 KiB/s rd, 25 KiB/s wr, 486 op/s 2024-03-29T05:02:38.620 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:38.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:38 smithi187 ceph-mon[93536]: failed to encode map e599 with expected crc 2024-03-29T05:02:38.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:38 smithi187 ceph-mon[93536]: failed to encode map e599 with expected crc 2024-03-29T05:02:38.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:38 smithi187 ceph-mon[93536]: osdmap e600: 8 total, 8 up, 8 in 2024-03-29T05:02:38.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:38 smithi187 ceph-mon[93536]: failed to encode map e600 with expected crc 2024-03-29T05:02:38.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:38 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/38681131' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:38.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:38 smithi181 ceph-mon[101970]: failed to encode map e599 with expected crc 2024-03-29T05:02:38.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:38 smithi181 ceph-mon[101970]: failed to encode map e599 with expected crc 2024-03-29T05:02:38.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:38 smithi181 ceph-mon[101970]: osdmap e600: 8 total, 8 up, 8 in 2024-03-29T05:02:38.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:38 smithi181 ceph-mon[101970]: failed to encode map e600 with expected crc 2024-03-29T05:02:38.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:38 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/38681131' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:38.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:38 smithi181 ceph-mon[106466]: failed to encode map e599 with expected crc 2024-03-29T05:02:38.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:38 smithi181 ceph-mon[106466]: failed to encode map e599 with expected crc 2024-03-29T05:02:38.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:38 smithi181 ceph-mon[106466]: osdmap e600: 8 total, 8 up, 8 in 2024-03-29T05:02:38.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:38 smithi181 ceph-mon[106466]: failed to encode map e600 with expected crc 2024-03-29T05:02:38.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:38 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/38681131' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:02:39.342 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:02:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:02:38] "GET /metrics HTTP/1.1" 200 45980 "" "Prometheus/2.43.0" 2024-03-29T05:02:39.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:39 smithi187 ceph-mon[93536]: failed to encode map e600 with expected crc 2024-03-29T05:02:39.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:39 smithi187 ceph-mon[93536]: failed to encode map e600 with expected crc 2024-03-29T05:02:39.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:39 smithi187 ceph-mon[93536]: failed to encode map e600 with expected crc 2024-03-29T05:02:39.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:39 smithi187 ceph-mon[93536]: failed to encode map e600 with expected crc 2024-03-29T05:02:39.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:39 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/38681131' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:39.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:39 smithi187 ceph-mon[93536]: osdmap e601: 8 total, 8 up, 8 in 2024-03-29T05:02:39.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:39 smithi187 ceph-mon[93536]: failed to encode map e601 with expected crc 2024-03-29T05:02:39.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:39 smithi187 ceph-mon[93536]: pgmap v575: 297 pgs: 5 creating+peering, 27 unknown, 265 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 511 KiB/s rd, 31 KiB/s wr, 611 op/s 2024-03-29T05:02:39.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:39 smithi181 ceph-mon[106466]: failed to encode map e600 with expected crc 2024-03-29T05:02:39.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:39 smithi181 ceph-mon[106466]: failed to encode map e600 with expected crc 2024-03-29T05:02:39.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:39 smithi181 ceph-mon[106466]: failed to encode map e600 with expected crc 2024-03-29T05:02:39.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:39 smithi181 ceph-mon[106466]: failed to encode map e600 with expected crc 2024-03-29T05:02:39.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:39 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/38681131' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:39.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:39 smithi181 ceph-mon[106466]: osdmap e601: 8 total, 8 up, 8 in 2024-03-29T05:02:39.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:39 smithi181 ceph-mon[106466]: failed to encode map e601 with expected crc 2024-03-29T05:02:39.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:39 smithi181 ceph-mon[106466]: pgmap v575: 297 pgs: 5 creating+peering, 27 unknown, 265 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 511 KiB/s rd, 31 KiB/s wr, 611 op/s 2024-03-29T05:02:39.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:39 smithi181 ceph-mon[101970]: failed to encode map e600 with expected crc 2024-03-29T05:02:39.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:39 smithi181 ceph-mon[101970]: failed to encode map e600 with expected crc 2024-03-29T05:02:39.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:39 smithi181 ceph-mon[101970]: failed to encode map e600 with expected crc 2024-03-29T05:02:39.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:39 smithi181 ceph-mon[101970]: failed to encode map e600 with expected crc 2024-03-29T05:02:39.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:39 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/38681131' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:02:39.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:39 smithi181 ceph-mon[101970]: osdmap e601: 8 total, 8 up, 8 in 2024-03-29T05:02:39.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:39 smithi181 ceph-mon[101970]: failed to encode map e601 with expected crc 2024-03-29T05:02:39.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:39 smithi181 ceph-mon[101970]: pgmap v575: 297 pgs: 5 creating+peering, 27 unknown, 265 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 511 KiB/s rd, 31 KiB/s wr, 611 op/s 2024-03-29T05:02:40.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:40 smithi187 ceph-mon[93536]: failed to encode map e601 with expected crc 2024-03-29T05:02:40.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:40 smithi187 ceph-mon[93536]: failed to encode map e601 with expected crc 2024-03-29T05:02:40.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:40 smithi187 ceph-mon[93536]: failed to encode map e601 with expected crc 2024-03-29T05:02:40.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:40 smithi187 ceph-mon[93536]: osdmap e602: 8 total, 8 up, 8 in 2024-03-29T05:02:40.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:40 smithi181 ceph-mon[101970]: failed to encode map e601 with expected crc 2024-03-29T05:02:40.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:40 smithi181 ceph-mon[101970]: failed to encode map e601 with expected crc 2024-03-29T05:02:40.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:40 smithi181 ceph-mon[101970]: failed to encode map e601 with expected crc 2024-03-29T05:02:40.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:40 smithi181 ceph-mon[101970]: osdmap e602: 8 total, 8 up, 8 in 2024-03-29T05:02:40.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:40 smithi181 ceph-mon[106466]: failed to encode map e601 with expected crc 2024-03-29T05:02:40.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:40 smithi181 ceph-mon[106466]: failed to encode map e601 with expected crc 2024-03-29T05:02:40.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:40 smithi181 ceph-mon[106466]: failed to encode map e601 with expected crc 2024-03-29T05:02:40.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:40 smithi181 ceph-mon[106466]: osdmap e602: 8 total, 8 up, 8 in 2024-03-29T05:02:40.893 INFO:tasks.workunit.client.0.smithi181.stdout:num images is: 1 2024-03-29T05:02:40.894 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 1 2024-03-29T05:02:40.894 INFO:tasks.workunit.client.0.smithi181.stdout:image: image34 2024-03-29T05:02:40.894 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image34 2024-03-29T05:02:40.919 INFO:tasks.workunit.client.0.smithi181.stdout:num images is: 2 2024-03-29T05:02:40.919 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 2 2024-03-29T05:02:40.919 INFO:tasks.workunit.client.0.smithi181.stdout:image: image34 2024-03-29T05:02:40.919 INFO:tasks.workunit.client.0.smithi181.stdout:image: image35 2024-03-29T05:02:40.919 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image34 2024-03-29T05:02:40.919 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image35 2024-03-29T05:02:40.971 INFO:tasks.workunit.client.0.smithi181.stdout:100.00% done 2024-03-29T05:02:40.971 INFO:tasks.workunit.client.0.smithi181.stdout:num images is: 3 2024-03-29T05:02:40.971 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 3 2024-03-29T05:02:40.972 INFO:tasks.workunit.client.0.smithi181.stdout:image: image34 2024-03-29T05:02:40.972 INFO:tasks.workunit.client.0.smithi181.stdout:image: image35 2024-03-29T05:02:40.972 INFO:tasks.workunit.client.0.smithi181.stdout:image: image36 2024-03-29T05:02:40.972 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image34 2024-03-29T05:02:40.972 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image35 2024-03-29T05:02:40.972 INFO:tasks.workunit.client.0.smithi181.stdout:expected = image36 2024-03-29T05:02:41.006 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestDeepCopyPP (4312 ms) 2024-03-29T05:02:41.006 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnap 2024-03-29T05:02:41.006 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:41.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:41 smithi187 ceph-mon[93536]: failed to encode map e602 with expected crc 2024-03-29T05:02:41.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:41 smithi187 ceph-mon[93536]: osdmap e603: 8 total, 8 up, 8 in 2024-03-29T05:02:41.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:41 smithi187 ceph-mon[93536]: pgmap v578: 297 pgs: 5 creating+peering, 27 unknown, 265 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 101 KiB/s rd, 6.2 KiB/s wr, 126 op/s 2024-03-29T05:02:41.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:41 smithi187 ceph-mon[93536]: failed to encode map e603 with expected crc 2024-03-29T05:02:41.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:02:41.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:41.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:41 smithi181 ceph-mon[101970]: failed to encode map e602 with expected crc 2024-03-29T05:02:41.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:41 smithi181 ceph-mon[101970]: osdmap e603: 8 total, 8 up, 8 in 2024-03-29T05:02:41.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:41 smithi181 ceph-mon[101970]: pgmap v578: 297 pgs: 5 creating+peering, 27 unknown, 265 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 101 KiB/s rd, 6.2 KiB/s wr, 126 op/s 2024-03-29T05:02:41.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:41 smithi181 ceph-mon[101970]: failed to encode map e603 with expected crc 2024-03-29T05:02:41.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:02:41.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:41.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:41 smithi181 ceph-mon[106466]: failed to encode map e602 with expected crc 2024-03-29T05:02:41.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:41 smithi181 ceph-mon[106466]: osdmap e603: 8 total, 8 up, 8 in 2024-03-29T05:02:41.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:41 smithi181 ceph-mon[106466]: pgmap v578: 297 pgs: 5 creating+peering, 27 unknown, 265 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 101 KiB/s rd, 6.2 KiB/s wr, 126 op/s 2024-03-29T05:02:41.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:41 smithi181 ceph-mon[106466]: failed to encode map e603 with expected crc 2024-03-29T05:02:41.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:02:41.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:42.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:42 smithi187 ceph-mon[93536]: failed to encode map e603 with expected crc 2024-03-29T05:02:42.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:42 smithi187 ceph-mon[93536]: failed to encode map e603 with expected crc 2024-03-29T05:02:42.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:42 smithi187 ceph-mon[93536]: osdmap e604: 8 total, 8 up, 8 in 2024-03-29T05:02:42.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:42 smithi181 ceph-mon[101970]: failed to encode map e603 with expected crc 2024-03-29T05:02:42.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:42 smithi181 ceph-mon[101970]: failed to encode map e603 with expected crc 2024-03-29T05:02:42.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:42 smithi181 ceph-mon[101970]: osdmap e604: 8 total, 8 up, 8 in 2024-03-29T05:02:42.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:42 smithi181 ceph-mon[106466]: failed to encode map e603 with expected crc 2024-03-29T05:02:42.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:42 smithi181 ceph-mon[106466]: failed to encode map e603 with expected crc 2024-03-29T05:02:42.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:42 smithi181 ceph-mon[106466]: osdmap e604: 8 total, 8 up, 8 in 2024-03-29T05:02:43.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:43 smithi187 ceph-mon[93536]: failed to encode map e604 with expected crc 2024-03-29T05:02:43.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:43 smithi187 ceph-mon[93536]: failed to encode map e604 with expected crc 2024-03-29T05:02:43.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:43 smithi187 ceph-mon[93536]: failed to encode map e604 with expected crc 2024-03-29T05:02:43.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:43 smithi187 ceph-mon[93536]: failed to encode map e605 with expected crc 2024-03-29T05:02:43.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:43 smithi187 ceph-mon[93536]: osdmap e605: 8 total, 8 up, 8 in 2024-03-29T05:02:43.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:43 smithi187 ceph-mon[93536]: pgmap v581: 297 pgs: 5 creating+peering, 292 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 121 KiB/s rd, 5.0 KiB/s wr, 140 op/s 2024-03-29T05:02:43.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:43 smithi181 ceph-mon[101970]: failed to encode map e604 with expected crc 2024-03-29T05:02:43.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:43 smithi181 ceph-mon[101970]: failed to encode map e604 with expected crc 2024-03-29T05:02:43.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:43 smithi181 ceph-mon[101970]: failed to encode map e604 with expected crc 2024-03-29T05:02:43.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:43 smithi181 ceph-mon[101970]: failed to encode map e605 with expected crc 2024-03-29T05:02:43.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:43 smithi181 ceph-mon[101970]: osdmap e605: 8 total, 8 up, 8 in 2024-03-29T05:02:43.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:43 smithi181 ceph-mon[101970]: pgmap v581: 297 pgs: 5 creating+peering, 292 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 121 KiB/s rd, 5.0 KiB/s wr, 140 op/s 2024-03-29T05:02:43.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:43 smithi181 ceph-mon[106466]: failed to encode map e604 with expected crc 2024-03-29T05:02:43.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:43 smithi181 ceph-mon[106466]: failed to encode map e604 with expected crc 2024-03-29T05:02:43.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:43 smithi181 ceph-mon[106466]: failed to encode map e604 with expected crc 2024-03-29T05:02:43.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:43 smithi181 ceph-mon[106466]: failed to encode map e605 with expected crc 2024-03-29T05:02:43.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:43 smithi181 ceph-mon[106466]: osdmap e605: 8 total, 8 up, 8 in 2024-03-29T05:02:43.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:43 smithi181 ceph-mon[106466]: pgmap v581: 297 pgs: 5 creating+peering, 292 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 121 KiB/s rd, 5.0 KiB/s wr, 140 op/s 2024-03-29T05:02:44.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:44 smithi187 ceph-mon[93536]: failed to encode map e605 with expected crc 2024-03-29T05:02:44.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:44 smithi187 ceph-mon[93536]: osdmap e606: 8 total, 8 up, 8 in 2024-03-29T05:02:44.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:44 smithi187 ceph-mon[93536]: failed to encode map e606 with expected crc 2024-03-29T05:02:44.645 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 1 2024-03-29T05:02:44.645 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 1 2024-03-29T05:02:44.645 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap1 2024-03-29T05:02:44.645 INFO:tasks.workunit.client.0.smithi181.stdout:found snap1 with size 2097152 2024-03-29T05:02:44.646 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 2 2024-03-29T05:02:44.646 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 2 2024-03-29T05:02:44.646 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap1 2024-03-29T05:02:44.646 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap2 2024-03-29T05:02:44.646 INFO:tasks.workunit.client.0.smithi181.stdout:found snap1 with size 2097152 2024-03-29T05:02:44.646 INFO:tasks.workunit.client.0.smithi181.stdout:found snap2 with size 4194304 2024-03-29T05:02:44.646 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 1 2024-03-29T05:02:44.646 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 1 2024-03-29T05:02:44.646 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap2 2024-03-29T05:02:44.646 INFO:tasks.workunit.client.0.smithi181.stdout:found snap2 with size 4194304 2024-03-29T05:02:44.646 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 0 2024-03-29T05:02:44.647 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 0 2024-03-29T05:02:44.647 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnap (3639 ms) 2024-03-29T05:02:44.647 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestGetSnapShotTimeStamp 2024-03-29T05:02:44.647 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:44.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:44 smithi181 ceph-mon[101970]: failed to encode map e605 with expected crc 2024-03-29T05:02:44.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:44 smithi181 ceph-mon[101970]: osdmap e606: 8 total, 8 up, 8 in 2024-03-29T05:02:44.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:44 smithi181 ceph-mon[101970]: failed to encode map e606 with expected crc 2024-03-29T05:02:44.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:44 smithi181 ceph-mon[106466]: failed to encode map e605 with expected crc 2024-03-29T05:02:44.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:44 smithi181 ceph-mon[106466]: osdmap e606: 8 total, 8 up, 8 in 2024-03-29T05:02:44.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:44 smithi181 ceph-mon[106466]: failed to encode map e606 with expected crc 2024-03-29T05:02:45.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:45 smithi187 ceph-mon[93536]: failed to encode map e606 with expected crc 2024-03-29T05:02:45.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:45 smithi187 ceph-mon[93536]: failed to encode map e606 with expected crc 2024-03-29T05:02:45.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:45 smithi187 ceph-mon[93536]: pgmap v584: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 408 KiB/s rd, 26 KiB/s wr, 482 op/s 2024-03-29T05:02:45.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:45 smithi187 ceph-mon[93536]: osdmap e607: 8 total, 8 up, 8 in 2024-03-29T05:02:45.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:45 smithi187 ceph-mon[93536]: failed to encode map e607 with expected crc 2024-03-29T05:02:45.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:45 smithi181 ceph-mon[106466]: failed to encode map e606 with expected crc 2024-03-29T05:02:45.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:45 smithi181 ceph-mon[106466]: failed to encode map e606 with expected crc 2024-03-29T05:02:45.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:45 smithi181 ceph-mon[106466]: pgmap v584: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 408 KiB/s rd, 26 KiB/s wr, 482 op/s 2024-03-29T05:02:45.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:45 smithi181 ceph-mon[106466]: osdmap e607: 8 total, 8 up, 8 in 2024-03-29T05:02:45.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:45 smithi181 ceph-mon[106466]: failed to encode map e607 with expected crc 2024-03-29T05:02:45.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:45 smithi181 ceph-mon[101970]: failed to encode map e606 with expected crc 2024-03-29T05:02:45.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:45 smithi181 ceph-mon[101970]: failed to encode map e606 with expected crc 2024-03-29T05:02:45.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:45 smithi181 ceph-mon[101970]: pgmap v584: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 408 KiB/s rd, 26 KiB/s wr, 482 op/s 2024-03-29T05:02:45.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:45 smithi181 ceph-mon[101970]: osdmap e607: 8 total, 8 up, 8 in 2024-03-29T05:02:45.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:45 smithi181 ceph-mon[101970]: failed to encode map e607 with expected crc 2024-03-29T05:02:46.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:46 smithi187 ceph-mon[93536]: failed to encode map e607 with expected crc 2024-03-29T05:02:46.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:46 smithi187 ceph-mon[93536]: failed to encode map e607 with expected crc 2024-03-29T05:02:46.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:46 smithi187 ceph-mon[93536]: osdmap e608: 8 total, 8 up, 8 in 2024-03-29T05:02:46.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:46 smithi187 ceph-mon[93536]: failed to encode map e608 with expected crc 2024-03-29T05:02:46.661 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestGetSnapShotTimeStamp (2016 ms) 2024-03-29T05:02:46.661 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnapPP 2024-03-29T05:02:46.662 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:46.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:46 smithi181 ceph-mon[106466]: failed to encode map e607 with expected crc 2024-03-29T05:02:46.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:46 smithi181 ceph-mon[106466]: failed to encode map e607 with expected crc 2024-03-29T05:02:46.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:46 smithi181 ceph-mon[106466]: osdmap e608: 8 total, 8 up, 8 in 2024-03-29T05:02:46.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:46 smithi181 ceph-mon[106466]: failed to encode map e608 with expected crc 2024-03-29T05:02:46.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:46 smithi181 ceph-mon[101970]: failed to encode map e607 with expected crc 2024-03-29T05:02:46.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:46 smithi181 ceph-mon[101970]: failed to encode map e607 with expected crc 2024-03-29T05:02:46.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:46 smithi181 ceph-mon[101970]: osdmap e608: 8 total, 8 up, 8 in 2024-03-29T05:02:46.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:46 smithi181 ceph-mon[101970]: failed to encode map e608 with expected crc 2024-03-29T05:02:47.629 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 1 2024-03-29T05:02:47.629 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 1 2024-03-29T05:02:47.629 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap1 2024-03-29T05:02:47.629 INFO:tasks.workunit.client.0.smithi181.stdout:found snap1 with size 2097152 2024-03-29T05:02:47.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:47 smithi187 ceph-mon[93536]: failed to encode map e608 with expected crc 2024-03-29T05:02:47.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:47 smithi187 ceph-mon[93536]: failed to encode map e608 with expected crc 2024-03-29T05:02:47.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:47 smithi187 ceph-mon[93536]: pgmap v586: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 289 KiB/s rd, 21 KiB/s wr, 346 op/s 2024-03-29T05:02:47.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:47 smithi187 ceph-mon[93536]: osdmap e609: 8 total, 8 up, 8 in 2024-03-29T05:02:47.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:47 smithi181 ceph-mon[106466]: failed to encode map e608 with expected crc 2024-03-29T05:02:47.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:47 smithi181 ceph-mon[106466]: failed to encode map e608 with expected crc 2024-03-29T05:02:47.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:47 smithi181 ceph-mon[106466]: pgmap v586: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 289 KiB/s rd, 21 KiB/s wr, 346 op/s 2024-03-29T05:02:47.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:47 smithi181 ceph-mon[106466]: osdmap e609: 8 total, 8 up, 8 in 2024-03-29T05:02:47.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:47 smithi181 ceph-mon[101970]: failed to encode map e608 with expected crc 2024-03-29T05:02:47.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:47 smithi181 ceph-mon[101970]: failed to encode map e608 with expected crc 2024-03-29T05:02:47.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:47 smithi181 ceph-mon[101970]: pgmap v586: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 289 KiB/s rd, 21 KiB/s wr, 346 op/s 2024-03-29T05:02:47.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:47 smithi181 ceph-mon[101970]: osdmap e609: 8 total, 8 up, 8 in 2024-03-29T05:02:48.631 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 2 2024-03-29T05:02:48.631 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 2 2024-03-29T05:02:48.631 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap1 2024-03-29T05:02:48.631 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap2 2024-03-29T05:02:48.631 INFO:tasks.workunit.client.0.smithi181.stdout:found snap1 with size 2097152 2024-03-29T05:02:48.631 INFO:tasks.workunit.client.0.smithi181.stdout:found snap2 with size 4194304 2024-03-29T05:02:48.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:48 smithi187 ceph-mon[93536]: failed to encode map e609 with expected crc 2024-03-29T05:02:48.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:48 smithi187 ceph-mon[93536]: failed to encode map e609 with expected crc 2024-03-29T05:02:48.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:48 smithi187 ceph-mon[93536]: osdmap e610: 8 total, 8 up, 8 in 2024-03-29T05:02:48.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:48 smithi181 ceph-mon[106466]: failed to encode map e609 with expected crc 2024-03-29T05:02:48.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:48 smithi181 ceph-mon[106466]: failed to encode map e609 with expected crc 2024-03-29T05:02:48.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:48 smithi181 ceph-mon[106466]: osdmap e610: 8 total, 8 up, 8 in 2024-03-29T05:02:48.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:48 smithi181 ceph-mon[101970]: failed to encode map e609 with expected crc 2024-03-29T05:02:48.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:48 smithi181 ceph-mon[101970]: failed to encode map e609 with expected crc 2024-03-29T05:02:48.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:48 smithi181 ceph-mon[101970]: osdmap e610: 8 total, 8 up, 8 in 2024-03-29T05:02:49.205 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:02:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:02:48] "GET /metrics HTTP/1.1" 200 50836 "" "Prometheus/2.43.0" 2024-03-29T05:02:49.633 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 1 2024-03-29T05:02:49.633 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 1 2024-03-29T05:02:49.633 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap2 2024-03-29T05:02:49.633 INFO:tasks.workunit.client.0.smithi181.stdout:found snap2 with size 4194304 2024-03-29T05:02:49.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:49 smithi187 ceph-mon[93536]: failed to encode map e610 with expected crc 2024-03-29T05:02:49.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:49 smithi187 ceph-mon[93536]: failed to encode map e610 with expected crc 2024-03-29T05:02:49.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:49 smithi187 ceph-mon[93536]: osdmap e611: 8 total, 8 up, 8 in 2024-03-29T05:02:49.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:49 smithi187 ceph-mon[93536]: pgmap v590: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 45 KiB/s rd, 5.0 KiB/s wr, 64 op/s 2024-03-29T05:02:49.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:49 smithi181 ceph-mon[106466]: failed to encode map e610 with expected crc 2024-03-29T05:02:49.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:49 smithi181 ceph-mon[106466]: failed to encode map e610 with expected crc 2024-03-29T05:02:49.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:49 smithi181 ceph-mon[106466]: osdmap e611: 8 total, 8 up, 8 in 2024-03-29T05:02:49.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:49 smithi181 ceph-mon[106466]: pgmap v590: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 45 KiB/s rd, 5.0 KiB/s wr, 64 op/s 2024-03-29T05:02:49.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:49 smithi181 ceph-mon[101970]: failed to encode map e610 with expected crc 2024-03-29T05:02:49.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:49 smithi181 ceph-mon[101970]: failed to encode map e610 with expected crc 2024-03-29T05:02:49.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:49 smithi181 ceph-mon[101970]: osdmap e611: 8 total, 8 up, 8 in 2024-03-29T05:02:49.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:49 smithi181 ceph-mon[101970]: pgmap v590: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 45 KiB/s rd, 5.0 KiB/s wr, 64 op/s 2024-03-29T05:02:50.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:50 smithi187 ceph-mon[93536]: failed to encode map e611 with expected crc 2024-03-29T05:02:50.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:50 smithi187 ceph-mon[93536]: failed to encode map e611 with expected crc 2024-03-29T05:02:50.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:50 smithi187 ceph-mon[93536]: failed to encode map e611 with expected crc 2024-03-29T05:02:50.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:50 smithi187 ceph-mon[93536]: osdmap e612: 8 total, 8 up, 8 in 2024-03-29T05:02:50.644 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 0 2024-03-29T05:02:50.644 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 0 2024-03-29T05:02:50.653 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnapPP (3992 ms) 2024-03-29T05:02:50.654 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestGetNameIdSnapPP 2024-03-29T05:02:50.654 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:50.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:50 smithi181 ceph-mon[106466]: failed to encode map e611 with expected crc 2024-03-29T05:02:50.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:50 smithi181 ceph-mon[106466]: failed to encode map e611 with expected crc 2024-03-29T05:02:50.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:50 smithi181 ceph-mon[106466]: failed to encode map e611 with expected crc 2024-03-29T05:02:50.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:50 smithi181 ceph-mon[106466]: osdmap e612: 8 total, 8 up, 8 in 2024-03-29T05:02:50.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:50 smithi181 ceph-mon[101970]: failed to encode map e611 with expected crc 2024-03-29T05:02:50.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:50 smithi181 ceph-mon[101970]: failed to encode map e611 with expected crc 2024-03-29T05:02:50.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:50 smithi181 ceph-mon[101970]: failed to encode map e611 with expected crc 2024-03-29T05:02:50.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:50 smithi181 ceph-mon[101970]: osdmap e612: 8 total, 8 up, 8 in 2024-03-29T05:02:51.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:51 smithi187 ceph-mon[93536]: failed to encode map e612 with expected crc 2024-03-29T05:02:51.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:51 smithi187 ceph-mon[93536]: failed to encode map e613 with expected crc 2024-03-29T05:02:51.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:51 smithi187 ceph-mon[93536]: pgmap v593: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 4.5 KiB/s wr, 59 op/s 2024-03-29T05:02:51.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:51 smithi187 ceph-mon[93536]: osdmap e613: 8 total, 8 up, 8 in 2024-03-29T05:02:51.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:51 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:51.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:51 smithi181 ceph-mon[106466]: failed to encode map e612 with expected crc 2024-03-29T05:02:51.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:51 smithi181 ceph-mon[106466]: failed to encode map e613 with expected crc 2024-03-29T05:02:51.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:51 smithi181 ceph-mon[106466]: pgmap v593: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 4.5 KiB/s wr, 59 op/s 2024-03-29T05:02:51.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:51 smithi181 ceph-mon[106466]: osdmap e613: 8 total, 8 up, 8 in 2024-03-29T05:02:51.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:51 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:51.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:51 smithi181 ceph-mon[101970]: failed to encode map e612 with expected crc 2024-03-29T05:02:51.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:51 smithi181 ceph-mon[101970]: failed to encode map e613 with expected crc 2024-03-29T05:02:51.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:51 smithi181 ceph-mon[101970]: pgmap v593: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 4.5 KiB/s wr, 59 op/s 2024-03-29T05:02:51.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:51 smithi181 ceph-mon[101970]: osdmap e613: 8 total, 8 up, 8 in 2024-03-29T05:02:51.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:51 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:02:52.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:52 smithi187 ceph-mon[93536]: failed to encode map e613 with expected crc 2024-03-29T05:02:52.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:52 smithi187 ceph-mon[93536]: failed to encode map e613 with expected crc 2024-03-29T05:02:52.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:52 smithi187 ceph-mon[93536]: osdmap e614: 8 total, 8 up, 8 in 2024-03-29T05:02:52.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:52 smithi187 ceph-mon[93536]: failed to encode map e614 with expected crc 2024-03-29T05:02:52.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:52 smithi187 ceph-mon[93536]: failed to encode map e614 with expected crc 2024-03-29T05:02:52.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:52 smithi181 ceph-mon[101970]: failed to encode map e613 with expected crc 2024-03-29T05:02:52.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:52 smithi181 ceph-mon[101970]: failed to encode map e613 with expected crc 2024-03-29T05:02:52.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:52 smithi181 ceph-mon[101970]: osdmap e614: 8 total, 8 up, 8 in 2024-03-29T05:02:52.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:52 smithi181 ceph-mon[101970]: failed to encode map e614 with expected crc 2024-03-29T05:02:52.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:52 smithi181 ceph-mon[101970]: failed to encode map e614 with expected crc 2024-03-29T05:02:52.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:52 smithi181 ceph-mon[106466]: failed to encode map e613 with expected crc 2024-03-29T05:02:52.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:52 smithi181 ceph-mon[106466]: failed to encode map e613 with expected crc 2024-03-29T05:02:52.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:52 smithi181 ceph-mon[106466]: osdmap e614: 8 total, 8 up, 8 in 2024-03-29T05:02:52.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:52 smithi181 ceph-mon[106466]: failed to encode map e614 with expected crc 2024-03-29T05:02:52.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:52 smithi181 ceph-mon[106466]: failed to encode map e614 with expected crc 2024-03-29T05:02:53.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:53 smithi187 ceph-mon[93536]: failed to encode map e614 with expected crc 2024-03-29T05:02:53.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:53 smithi187 ceph-mon[93536]: failed to encode map e614 with expected crc 2024-03-29T05:02:53.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:53 smithi187 ceph-mon[93536]: osdmap e615: 8 total, 8 up, 8 in 2024-03-29T05:02:53.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:53 smithi187 ceph-mon[93536]: pgmap v596: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 6.0 KiB/s wr, 72 op/s 2024-03-29T05:02:53.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:53 smithi187 ceph-mon[93536]: failed to encode map e615 with expected crc 2024-03-29T05:02:53.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:53 smithi181 ceph-mon[106466]: failed to encode map e614 with expected crc 2024-03-29T05:02:53.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:53 smithi181 ceph-mon[106466]: failed to encode map e614 with expected crc 2024-03-29T05:02:53.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:53 smithi181 ceph-mon[106466]: osdmap e615: 8 total, 8 up, 8 in 2024-03-29T05:02:53.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:53 smithi181 ceph-mon[106466]: pgmap v596: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 6.0 KiB/s wr, 72 op/s 2024-03-29T05:02:53.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:53 smithi181 ceph-mon[106466]: failed to encode map e615 with expected crc 2024-03-29T05:02:53.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:53 smithi181 ceph-mon[101970]: failed to encode map e614 with expected crc 2024-03-29T05:02:53.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:53 smithi181 ceph-mon[101970]: failed to encode map e614 with expected crc 2024-03-29T05:02:53.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:53 smithi181 ceph-mon[101970]: osdmap e615: 8 total, 8 up, 8 in 2024-03-29T05:02:53.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:53 smithi181 ceph-mon[101970]: pgmap v596: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 6.0 KiB/s wr, 72 op/s 2024-03-29T05:02:53.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:53 smithi181 ceph-mon[101970]: failed to encode map e615 with expected crc 2024-03-29T05:02:54.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:54 smithi187 ceph-mon[93536]: failed to encode map e615 with expected crc 2024-03-29T05:02:54.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:54 smithi187 ceph-mon[93536]: failed to encode map e615 with expected crc 2024-03-29T05:02:54.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:54 smithi187 ceph-mon[93536]: failed to encode map e616 with expected crc 2024-03-29T05:02:54.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:54 smithi187 ceph-mon[93536]: osdmap e616: 8 total, 8 up, 8 in 2024-03-29T05:02:54.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:54 smithi187 ceph-mon[93536]: failed to encode map e616 with expected crc 2024-03-29T05:02:54.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:54 smithi181 ceph-mon[106466]: failed to encode map e615 with expected crc 2024-03-29T05:02:54.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:54 smithi181 ceph-mon[106466]: failed to encode map e615 with expected crc 2024-03-29T05:02:54.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:54 smithi181 ceph-mon[106466]: failed to encode map e616 with expected crc 2024-03-29T05:02:54.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:54 smithi181 ceph-mon[106466]: osdmap e616: 8 total, 8 up, 8 in 2024-03-29T05:02:54.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:54 smithi181 ceph-mon[106466]: failed to encode map e616 with expected crc 2024-03-29T05:02:54.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:54 smithi181 ceph-mon[101970]: failed to encode map e615 with expected crc 2024-03-29T05:02:54.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:54 smithi181 ceph-mon[101970]: failed to encode map e615 with expected crc 2024-03-29T05:02:54.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:54 smithi181 ceph-mon[101970]: failed to encode map e616 with expected crc 2024-03-29T05:02:54.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:54 smithi181 ceph-mon[101970]: osdmap e616: 8 total, 8 up, 8 in 2024-03-29T05:02:54.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:54 smithi181 ceph-mon[101970]: failed to encode map e616 with expected crc 2024-03-29T05:02:55.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:55 smithi187 ceph-mon[93536]: failed to encode map e616 with expected crc 2024-03-29T05:02:55.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:55 smithi187 ceph-mon[93536]: osdmap e617: 8 total, 8 up, 8 in 2024-03-29T05:02:55.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:55 smithi187 ceph-mon[93536]: failed to encode map e617 with expected crc 2024-03-29T05:02:55.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:55 smithi187 ceph-mon[93536]: pgmap v599: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 6.2 KiB/s wr, 73 op/s 2024-03-29T05:02:55.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:55 smithi181 ceph-mon[106466]: failed to encode map e616 with expected crc 2024-03-29T05:02:55.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:55 smithi181 ceph-mon[106466]: osdmap e617: 8 total, 8 up, 8 in 2024-03-29T05:02:55.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:55 smithi181 ceph-mon[106466]: failed to encode map e617 with expected crc 2024-03-29T05:02:55.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:55 smithi181 ceph-mon[106466]: pgmap v599: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 6.2 KiB/s wr, 73 op/s 2024-03-29T05:02:55.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:55 smithi181 ceph-mon[101970]: failed to encode map e616 with expected crc 2024-03-29T05:02:55.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:55 smithi181 ceph-mon[101970]: osdmap e617: 8 total, 8 up, 8 in 2024-03-29T05:02:55.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:55 smithi181 ceph-mon[101970]: failed to encode map e617 with expected crc 2024-03-29T05:02:55.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:55 smithi181 ceph-mon[101970]: pgmap v599: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 6.2 KiB/s wr, 73 op/s 2024-03-29T05:02:56.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:56 smithi187 ceph-mon[93536]: failed to encode map e617 with expected crc 2024-03-29T05:02:56.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:56 smithi187 ceph-mon[93536]: osdmap e618: 8 total, 8 up, 8 in 2024-03-29T05:02:56.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:56 smithi187 ceph-mon[93536]: failed to encode map e618 with expected crc 2024-03-29T05:02:56.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:56 smithi187 ceph-mon[93536]: failed to encode map e618 with expected crc 2024-03-29T05:02:56.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:56 smithi187 ceph-mon[93536]: failed to encode map e618 with expected crc 2024-03-29T05:02:56.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:02:56.653 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 0 2024-03-29T05:02:56.653 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 0 2024-03-29T05:02:56.658 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestGetNameIdSnapPP (6006 ms) 2024-03-29T05:02:56.659 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCreateLsRenameSnapPP 2024-03-29T05:02:56.659 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:02:56.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:56 smithi181 ceph-mon[106466]: failed to encode map e617 with expected crc 2024-03-29T05:02:56.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:56 smithi181 ceph-mon[106466]: osdmap e618: 8 total, 8 up, 8 in 2024-03-29T05:02:56.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:56 smithi181 ceph-mon[106466]: failed to encode map e618 with expected crc 2024-03-29T05:02:56.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:56 smithi181 ceph-mon[106466]: failed to encode map e618 with expected crc 2024-03-29T05:02:56.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:56 smithi181 ceph-mon[106466]: failed to encode map e618 with expected crc 2024-03-29T05:02:56.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:02:56.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:56 smithi181 ceph-mon[101970]: failed to encode map e617 with expected crc 2024-03-29T05:02:56.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:56 smithi181 ceph-mon[101970]: osdmap e618: 8 total, 8 up, 8 in 2024-03-29T05:02:56.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:56 smithi181 ceph-mon[101970]: failed to encode map e618 with expected crc 2024-03-29T05:02:56.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:56 smithi181 ceph-mon[101970]: failed to encode map e618 with expected crc 2024-03-29T05:02:56.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:56 smithi181 ceph-mon[101970]: failed to encode map e618 with expected crc 2024-03-29T05:02:56.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:02:57.630 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 1 2024-03-29T05:02:57.630 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 1 2024-03-29T05:02:57.630 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap1 2024-03-29T05:02:57.630 INFO:tasks.workunit.client.0.smithi181.stdout:found snap1 with size 2097152 2024-03-29T05:02:57.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:57 smithi187 ceph-mon[93536]: failed to encode map e618 with expected crc 2024-03-29T05:02:57.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:57 smithi187 ceph-mon[93536]: pgmap v601: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1019 B/s rd, 764 B/s wr, 3 op/s 2024-03-29T05:02:57.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:57 smithi187 ceph-mon[93536]: osdmap e619: 8 total, 8 up, 8 in 2024-03-29T05:02:57.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:57 smithi187 ceph-mon[93536]: failed to encode map e619 with expected crc 2024-03-29T05:02:57.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:57 smithi187 ceph-mon[93536]: failed to encode map e619 with expected crc 2024-03-29T05:02:57.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:57 smithi181 ceph-mon[106466]: failed to encode map e618 with expected crc 2024-03-29T05:02:57.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:57 smithi181 ceph-mon[106466]: pgmap v601: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1019 B/s rd, 764 B/s wr, 3 op/s 2024-03-29T05:02:57.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:57 smithi181 ceph-mon[106466]: osdmap e619: 8 total, 8 up, 8 in 2024-03-29T05:02:57.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:57 smithi181 ceph-mon[106466]: failed to encode map e619 with expected crc 2024-03-29T05:02:57.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:57 smithi181 ceph-mon[106466]: failed to encode map e619 with expected crc 2024-03-29T05:02:57.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:57 smithi181 ceph-mon[101970]: failed to encode map e618 with expected crc 2024-03-29T05:02:57.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:57 smithi181 ceph-mon[101970]: pgmap v601: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1019 B/s rd, 764 B/s wr, 3 op/s 2024-03-29T05:02:57.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:57 smithi181 ceph-mon[101970]: osdmap e619: 8 total, 8 up, 8 in 2024-03-29T05:02:57.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:57 smithi181 ceph-mon[101970]: failed to encode map e619 with expected crc 2024-03-29T05:02:57.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:57 smithi181 ceph-mon[101970]: failed to encode map e619 with expected crc 2024-03-29T05:02:58.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:58 smithi187 ceph-mon[93536]: failed to encode map e619 with expected crc 2024-03-29T05:02:58.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:58 smithi187 ceph-mon[93536]: failed to encode map e620 with expected crc 2024-03-29T05:02:58.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:58 smithi187 ceph-mon[93536]: osdmap e620: 8 total, 8 up, 8 in 2024-03-29T05:02:58.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:58 smithi187 ceph-mon[93536]: failed to encode map e620 with expected crc 2024-03-29T05:02:58.636 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 2 2024-03-29T05:02:58.636 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 2 2024-03-29T05:02:58.636 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap1 2024-03-29T05:02:58.636 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap2 2024-03-29T05:02:58.636 INFO:tasks.workunit.client.0.smithi181.stdout:found snap1 with size 2097152 2024-03-29T05:02:58.636 INFO:tasks.workunit.client.0.smithi181.stdout:found snap2 with size 4194304 2024-03-29T05:02:58.640 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 2 2024-03-29T05:02:58.640 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 2 2024-03-29T05:02:58.640 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap1-rename 2024-03-29T05:02:58.640 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap2 2024-03-29T05:02:58.640 INFO:tasks.workunit.client.0.smithi181.stdout:found snap1-rename with size 2097152 2024-03-29T05:02:58.640 INFO:tasks.workunit.client.0.smithi181.stdout:found snap2 with size 4194304 2024-03-29T05:02:58.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:58 smithi181 ceph-mon[106466]: failed to encode map e619 with expected crc 2024-03-29T05:02:58.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:58 smithi181 ceph-mon[106466]: failed to encode map e620 with expected crc 2024-03-29T05:02:58.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:58 smithi181 ceph-mon[106466]: osdmap e620: 8 total, 8 up, 8 in 2024-03-29T05:02:58.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:58 smithi181 ceph-mon[106466]: failed to encode map e620 with expected crc 2024-03-29T05:02:58.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:58 smithi181 ceph-mon[101970]: failed to encode map e619 with expected crc 2024-03-29T05:02:58.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:58 smithi181 ceph-mon[101970]: failed to encode map e620 with expected crc 2024-03-29T05:02:58.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:58 smithi181 ceph-mon[101970]: osdmap e620: 8 total, 8 up, 8 in 2024-03-29T05:02:58.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:58 smithi181 ceph-mon[101970]: failed to encode map e620 with expected crc 2024-03-29T05:02:59.205 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:02:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:02:58] "GET /metrics HTTP/1.1" 200 50885 "" "Prometheus/2.43.0" 2024-03-29T05:02:59.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:59 smithi187 ceph-mon[93536]: failed to encode map e620 with expected crc 2024-03-29T05:02:59.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:59 smithi187 ceph-mon[93536]: osdmap e621: 8 total, 8 up, 8 in 2024-03-29T05:02:59.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:59 smithi187 ceph-mon[93536]: failed to encode map e621 with expected crc 2024-03-29T05:02:59.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:02:59 smithi187 ceph-mon[93536]: pgmap v605: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 45 KiB/s rd, 5.2 KiB/s wr, 62 op/s 2024-03-29T05:02:59.641 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 1 2024-03-29T05:02:59.641 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 1 2024-03-29T05:02:59.641 INFO:tasks.workunit.client.0.smithi181.stdout:snap: snap2-rename 2024-03-29T05:02:59.641 INFO:tasks.workunit.client.0.smithi181.stdout:found snap2-rename with size 4194304 2024-03-29T05:02:59.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:59 smithi181 ceph-mon[101970]: failed to encode map e620 with expected crc 2024-03-29T05:02:59.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:59 smithi181 ceph-mon[101970]: osdmap e621: 8 total, 8 up, 8 in 2024-03-29T05:02:59.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:59 smithi181 ceph-mon[101970]: failed to encode map e621 with expected crc 2024-03-29T05:02:59.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:02:59 smithi181 ceph-mon[101970]: pgmap v605: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 45 KiB/s rd, 5.2 KiB/s wr, 62 op/s 2024-03-29T05:02:59.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:59 smithi181 ceph-mon[106466]: failed to encode map e620 with expected crc 2024-03-29T05:02:59.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:59 smithi181 ceph-mon[106466]: osdmap e621: 8 total, 8 up, 8 in 2024-03-29T05:02:59.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:59 smithi181 ceph-mon[106466]: failed to encode map e621 with expected crc 2024-03-29T05:02:59.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:02:59 smithi181 ceph-mon[106466]: pgmap v605: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 45 KiB/s rd, 5.2 KiB/s wr, 62 op/s 2024-03-29T05:03:00.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:00 smithi187 ceph-mon[93536]: failed to encode map e621 with expected crc 2024-03-29T05:03:00.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:00 smithi187 ceph-mon[93536]: osdmap e622: 8 total, 8 up, 8 in 2024-03-29T05:03:00.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:00 smithi187 ceph-mon[93536]: failed to encode map e622 with expected crc 2024-03-29T05:03:00.637 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 0 2024-03-29T05:03:00.637 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 0 2024-03-29T05:03:00.644 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCreateLsRenameSnapPP (3985 ms) 2024-03-29T05:03:00.644 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ConcurrentCreatesUnvalidatedPool 2024-03-29T05:03:00.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:00 smithi181 ceph-mon[101970]: failed to encode map e621 with expected crc 2024-03-29T05:03:00.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:00 smithi181 ceph-mon[101970]: osdmap e622: 8 total, 8 up, 8 in 2024-03-29T05:03:00.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:00 smithi181 ceph-mon[101970]: failed to encode map e622 with expected crc 2024-03-29T05:03:00.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:00 smithi181 ceph-mon[106466]: failed to encode map e621 with expected crc 2024-03-29T05:03:00.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:00 smithi181 ceph-mon[106466]: osdmap e622: 8 total, 8 up, 8 in 2024-03-29T05:03:00.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:00 smithi181 ceph-mon[106466]: failed to encode map e622 with expected crc 2024-03-29T05:03:01.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:01 smithi181 ceph-mon[106466]: failed to encode map e622 with expected crc 2024-03-29T05:03:01.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:01 smithi181 ceph-mon[106466]: osdmap e623: 8 total, 8 up, 8 in 2024-03-29T05:03:01.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:01 smithi181 ceph-mon[106466]: pgmap v608: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 4.8 KiB/s wr, 54 op/s 2024-03-29T05:03:01.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:01 smithi181 ceph-mon[106466]: failed to encode map e623 with expected crc 2024-03-29T05:03:01.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:01 smithi181 ceph-mon[101970]: failed to encode map e622 with expected crc 2024-03-29T05:03:01.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:01 smithi181 ceph-mon[101970]: osdmap e623: 8 total, 8 up, 8 in 2024-03-29T05:03:01.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:01 smithi181 ceph-mon[101970]: pgmap v608: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 4.8 KiB/s wr, 54 op/s 2024-03-29T05:03:01.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:01 smithi181 ceph-mon[101970]: failed to encode map e623 with expected crc 2024-03-29T05:03:01.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:01 smithi187 ceph-mon[93536]: failed to encode map e622 with expected crc 2024-03-29T05:03:01.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:01 smithi187 ceph-mon[93536]: osdmap e623: 8 total, 8 up, 8 in 2024-03-29T05:03:01.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:01 smithi187 ceph-mon[93536]: pgmap v608: 297 pgs: 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 4.8 KiB/s wr, 54 op/s 2024-03-29T05:03:01.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:01 smithi187 ceph-mon[93536]: failed to encode map e623 with expected crc 2024-03-29T05:03:02.624 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:03:02.625 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:03:02.625 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:03:02.625 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:03:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[101970]: failed to encode map e623 with expected crc 2024-03-29T05:03:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[101970]: failed to encode map e623 with expected crc 2024-03-29T05:03:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[101970]: failed to encode map e624 with expected crc 2024-03-29T05:03:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[101970]: osdmap e624: 8 total, 8 up, 8 in 2024-03-29T05:03:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[101970]: failed to encode map e624 with expected crc 2024-03-29T05:03:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[101970]: failed to encode map e624 with expected crc 2024-03-29T05:03:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2775425780' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[101970]: failed to encode map e624 with expected crc 2024-03-29T05:03:02.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[101970]: failed to encode map e624 with expected crc 2024-03-29T05:03:02.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[106466]: failed to encode map e623 with expected crc 2024-03-29T05:03:02.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[106466]: failed to encode map e623 with expected crc 2024-03-29T05:03:02.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[106466]: failed to encode map e624 with expected crc 2024-03-29T05:03:02.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[106466]: osdmap e624: 8 total, 8 up, 8 in 2024-03-29T05:03:02.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[106466]: failed to encode map e624 with expected crc 2024-03-29T05:03:02.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[106466]: failed to encode map e624 with expected crc 2024-03-29T05:03:02.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2775425780' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:02.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[106466]: failed to encode map e624 with expected crc 2024-03-29T05:03:02.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:02 smithi181 ceph-mon[106466]: failed to encode map e624 with expected crc 2024-03-29T05:03:02.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:02 smithi187 ceph-mon[93536]: failed to encode map e623 with expected crc 2024-03-29T05:03:02.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:02 smithi187 ceph-mon[93536]: failed to encode map e623 with expected crc 2024-03-29T05:03:02.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:02 smithi187 ceph-mon[93536]: failed to encode map e624 with expected crc 2024-03-29T05:03:02.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:02 smithi187 ceph-mon[93536]: osdmap e624: 8 total, 8 up, 8 in 2024-03-29T05:03:02.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:02 smithi187 ceph-mon[93536]: failed to encode map e624 with expected crc 2024-03-29T05:03:02.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:02 smithi187 ceph-mon[93536]: failed to encode map e624 with expected crc 2024-03-29T05:03:02.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:02 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2775425780' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:02.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:02 smithi187 ceph-mon[93536]: failed to encode map e624 with expected crc 2024-03-29T05:03:02.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:02 smithi187 ceph-mon[93536]: failed to encode map e624 with expected crc 2024-03-29T05:03:03.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[101970]: failed to encode map e624 with expected crc 2024-03-29T05:03:03.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[101970]: failed to encode map e624 with expected crc 2024-03-29T05:03:03.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[101970]: failed to encode map e624 with expected crc 2024-03-29T05:03:03.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[101970]: failed to encode map e624 with expected crc 2024-03-29T05:03:03.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2775425780' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:03:03.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[101970]: failed to encode map e625 with expected crc 2024-03-29T05:03:03.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[101970]: osdmap e625: 8 total, 8 up, 8 in 2024-03-29T05:03:03.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[101970]: failed to encode map e625 with expected crc 2024-03-29T05:03:03.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[101970]: failed to encode map e625 with expected crc 2024-03-29T05:03:03.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[101970]: pgmap v611: 329 pgs: 9 creating+peering, 23 unknown, 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 4.7 KiB/s wr, 70 op/s 2024-03-29T05:03:03.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[106466]: failed to encode map e624 with expected crc 2024-03-29T05:03:03.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[106466]: failed to encode map e624 with expected crc 2024-03-29T05:03:03.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[106466]: failed to encode map e624 with expected crc 2024-03-29T05:03:03.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[106466]: failed to encode map e624 with expected crc 2024-03-29T05:03:03.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2775425780' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:03:03.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[106466]: failed to encode map e625 with expected crc 2024-03-29T05:03:03.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[106466]: osdmap e625: 8 total, 8 up, 8 in 2024-03-29T05:03:03.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[106466]: failed to encode map e625 with expected crc 2024-03-29T05:03:03.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[106466]: failed to encode map e625 with expected crc 2024-03-29T05:03:03.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:03 smithi181 ceph-mon[106466]: pgmap v611: 329 pgs: 9 creating+peering, 23 unknown, 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 4.7 KiB/s wr, 70 op/s 2024-03-29T05:03:03.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:03 smithi187 ceph-mon[93536]: failed to encode map e624 with expected crc 2024-03-29T05:03:03.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:03 smithi187 ceph-mon[93536]: failed to encode map e624 with expected crc 2024-03-29T05:03:03.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:03 smithi187 ceph-mon[93536]: failed to encode map e624 with expected crc 2024-03-29T05:03:03.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:03 smithi187 ceph-mon[93536]: failed to encode map e624 with expected crc 2024-03-29T05:03:03.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:03 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2775425780' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:03:03.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:03 smithi187 ceph-mon[93536]: failed to encode map e625 with expected crc 2024-03-29T05:03:03.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:03 smithi187 ceph-mon[93536]: osdmap e625: 8 total, 8 up, 8 in 2024-03-29T05:03:03.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:03 smithi187 ceph-mon[93536]: failed to encode map e625 with expected crc 2024-03-29T05:03:03.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:03 smithi187 ceph-mon[93536]: failed to encode map e625 with expected crc 2024-03-29T05:03:03.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:03 smithi187 ceph-mon[93536]: pgmap v611: 329 pgs: 9 creating+peering, 23 unknown, 297 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 4.7 KiB/s wr, 70 op/s 2024-03-29T05:03:04.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:04 smithi181 ceph-mon[101970]: failed to encode map e625 with expected crc 2024-03-29T05:03:04.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:04 smithi181 ceph-mon[101970]: failed to encode map e626 with expected crc 2024-03-29T05:03:04.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:04 smithi181 ceph-mon[101970]: osdmap e626: 8 total, 8 up, 8 in 2024-03-29T05:03:04.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:04 smithi181 ceph-mon[106466]: failed to encode map e625 with expected crc 2024-03-29T05:03:04.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:04 smithi181 ceph-mon[106466]: failed to encode map e626 with expected crc 2024-03-29T05:03:04.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:04 smithi181 ceph-mon[106466]: osdmap e626: 8 total, 8 up, 8 in 2024-03-29T05:03:04.885 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ConcurrentCreatesUnvalidatedPool (4241 ms) 2024-03-29T05:03:04.885 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.RemoveFullTry 2024-03-29T05:03:04.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:04 smithi187 ceph-mon[93536]: failed to encode map e625 with expected crc 2024-03-29T05:03:04.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:04 smithi187 ceph-mon[93536]: failed to encode map e626 with expected crc 2024-03-29T05:03:04.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:04 smithi187 ceph-mon[93536]: osdmap e626: 8 total, 8 up, 8 in 2024-03-29T05:03:05.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:05 smithi181 ceph-mon[101970]: failed to encode map e626 with expected crc 2024-03-29T05:03:05.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:05 smithi181 ceph-mon[101970]: osdmap e627: 8 total, 8 up, 8 in 2024-03-29T05:03:05.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:05 smithi181 ceph-mon[101970]: pgmap v614: 329 pgs: 9 creating+peering, 18 unknown, 302 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 4.7 KiB/s wr, 72 op/s 2024-03-29T05:03:05.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:05 smithi181 ceph-mon[101970]: failed to encode map e627 with expected crc 2024-03-29T05:03:05.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:05 smithi181 ceph-mon[101970]: failed to encode map e627 with expected crc 2024-03-29T05:03:05.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:05 smithi181 ceph-mon[106466]: failed to encode map e626 with expected crc 2024-03-29T05:03:05.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:05 smithi181 ceph-mon[106466]: osdmap e627: 8 total, 8 up, 8 in 2024-03-29T05:03:05.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:05 smithi181 ceph-mon[106466]: pgmap v614: 329 pgs: 9 creating+peering, 18 unknown, 302 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 4.7 KiB/s wr, 72 op/s 2024-03-29T05:03:05.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:05 smithi181 ceph-mon[106466]: failed to encode map e627 with expected crc 2024-03-29T05:03:05.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:05 smithi181 ceph-mon[106466]: failed to encode map e627 with expected crc 2024-03-29T05:03:05.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:05 smithi187 ceph-mon[93536]: failed to encode map e626 with expected crc 2024-03-29T05:03:05.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:05 smithi187 ceph-mon[93536]: osdmap e627: 8 total, 8 up, 8 in 2024-03-29T05:03:05.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:05 smithi187 ceph-mon[93536]: pgmap v614: 329 pgs: 9 creating+peering, 18 unknown, 302 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 4.7 KiB/s wr, 72 op/s 2024-03-29T05:03:05.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:05 smithi187 ceph-mon[93536]: failed to encode map e627 with expected crc 2024-03-29T05:03:05.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:05 smithi187 ceph-mon[93536]: failed to encode map e627 with expected crc 2024-03-29T05:03:06.673 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:03:06.679 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-mon[106466]: failed to encode map e627 with expected crc 2024-03-29T05:03:06.679 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-mon[106466]: osdmap e628: 8 total, 8 up, 8 in 2024-03-29T05:03:06.679 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-mon[106466]: failed to encode map e628 with expected crc 2024-03-29T05:03:06.680 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3996942524' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:06.680 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:06.680 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-mon[106466]: failed to encode map e628 with expected crc 2024-03-29T05:03:06.680 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-mon[106466]: failed to encode map e628 with expected crc 2024-03-29T05:03:06.680 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-mon[106466]: failed to encode map e628 with expected crc 2024-03-29T05:03:06.680 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-mon[101970]: failed to encode map e627 with expected crc 2024-03-29T05:03:06.680 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-mon[101970]: osdmap e628: 8 total, 8 up, 8 in 2024-03-29T05:03:06.680 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-mon[101970]: failed to encode map e628 with expected crc 2024-03-29T05:03:06.680 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3996942524' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:06.680 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:06.681 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-mon[101970]: failed to encode map e628 with expected crc 2024-03-29T05:03:06.681 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-mon[101970]: failed to encode map e628 with expected crc 2024-03-29T05:03:06.681 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-mon[101970]: failed to encode map e628 with expected crc 2024-03-29T05:03:06.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:06 smithi187 ceph-mon[93536]: failed to encode map e627 with expected crc 2024-03-29T05:03:06.947 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:06 smithi187 ceph-mon[93536]: osdmap e628: 8 total, 8 up, 8 in 2024-03-29T05:03:06.947 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:06 smithi187 ceph-mon[93536]: failed to encode map e628 with expected crc 2024-03-29T05:03:06.947 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:06 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3996942524' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:06.947 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:06 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:06.948 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:06 smithi187 ceph-mon[93536]: failed to encode map e628 with expected crc 2024-03-29T05:03:06.948 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:06 smithi187 ceph-mon[93536]: failed to encode map e628 with expected crc 2024-03-29T05:03:06.948 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:06 smithi187 ceph-mon[93536]: failed to encode map e628 with expected crc 2024-03-29T05:03:06.955 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:03:06] ENGINE Bus STOPPING 2024-03-29T05:03:06.955 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:03:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T05:03:06.955 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:03:06] ENGINE Bus STOPPED 2024-03-29T05:03:06.955 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:03:06] ENGINE Bus STARTING 2024-03-29T05:03:06.955 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:03:06] ENGINE Serving on http://:::9283 2024-03-29T05:03:06.955 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:06 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:03:06] ENGINE Bus STARTED 2024-03-29T05:03:07.670 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:07 smithi181 ceph-mon[106466]: failed to encode map e628 with expected crc 2024-03-29T05:03:07.670 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:07 smithi181 ceph-mon[106466]: pgmap v616: 361 pgs: 22 creating+peering, 23 unknown, 316 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 2.5 KiB/s wr, 65 op/s 2024-03-29T05:03:07.670 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:07 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:03:07.671 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:07 smithi181 ceph-mon[106466]: osdmap e629: 8 total, 8 up, 8 in 2024-03-29T05:03:07.671 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:07 smithi181 ceph-mon[106466]: failed to encode map e629 with expected crc 2024-03-29T05:03:07.671 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:07 smithi181 ceph-mon[106466]: failed to encode map e629 with expected crc 2024-03-29T05:03:07.671 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:07 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' 2024-03-29T05:03:07.671 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:07 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:03:07.671 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:07 smithi181 ceph-mon[101970]: failed to encode map e628 with expected crc 2024-03-29T05:03:07.671 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:07 smithi181 ceph-mon[101970]: pgmap v616: 361 pgs: 22 creating+peering, 23 unknown, 316 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 2.5 KiB/s wr, 65 op/s 2024-03-29T05:03:07.671 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:07 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:03:07.671 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:07 smithi181 ceph-mon[101970]: osdmap e629: 8 total, 8 up, 8 in 2024-03-29T05:03:07.671 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:07 smithi181 ceph-mon[101970]: failed to encode map e629 with expected crc 2024-03-29T05:03:07.671 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:07 smithi181 ceph-mon[101970]: failed to encode map e629 with expected crc 2024-03-29T05:03:07.671 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:07 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' 2024-03-29T05:03:07.672 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:07 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:03:07.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:07 smithi187 ceph-mon[93536]: failed to encode map e628 with expected crc 2024-03-29T05:03:07.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:07 smithi187 ceph-mon[93536]: pgmap v616: 361 pgs: 22 creating+peering, 23 unknown, 316 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 2.5 KiB/s wr, 65 op/s 2024-03-29T05:03:07.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:07 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:03:07.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:07 smithi187 ceph-mon[93536]: osdmap e629: 8 total, 8 up, 8 in 2024-03-29T05:03:07.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:07 smithi187 ceph-mon[93536]: failed to encode map e629 with expected crc 2024-03-29T05:03:07.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:07 smithi187 ceph-mon[93536]: failed to encode map e629 with expected crc 2024-03-29T05:03:07.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:07 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' 2024-03-29T05:03:07.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:07 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:03:08.685 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:08 smithi181 ceph-mon[106466]: failed to encode map e629 with expected crc 2024-03-29T05:03:08.685 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:08 smithi181 ceph-mon[106466]: failed to encode map e629 with expected crc 2024-03-29T05:03:08.685 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:08 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:03:08.685 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:08 smithi181 ceph-mon[106466]: osdmap e630: 8 total, 8 up, 8 in 2024-03-29T05:03:08.685 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:08 smithi181 ceph-mon[106466]: failed to encode map e630 with expected crc 2024-03-29T05:03:08.685 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:08 smithi181 ceph-mon[106466]: failed to encode map e630 with expected crc 2024-03-29T05:03:08.685 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:08 smithi181 ceph-mon[101970]: failed to encode map e629 with expected crc 2024-03-29T05:03:08.686 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:08 smithi181 ceph-mon[101970]: failed to encode map e629 with expected crc 2024-03-29T05:03:08.686 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:08 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:03:08.686 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:08 smithi181 ceph-mon[101970]: osdmap e630: 8 total, 8 up, 8 in 2024-03-29T05:03:08.686 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:08 smithi181 ceph-mon[101970]: failed to encode map e630 with expected crc 2024-03-29T05:03:08.686 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:08 smithi181 ceph-mon[101970]: failed to encode map e630 with expected crc 2024-03-29T05:03:08.691 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:03:08.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:08 smithi187 ceph-mon[93536]: failed to encode map e629 with expected crc 2024-03-29T05:03:08.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:08 smithi187 ceph-mon[93536]: failed to encode map e629 with expected crc 2024-03-29T05:03:08.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:08 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:03:08.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:08 smithi187 ceph-mon[93536]: osdmap e630: 8 total, 8 up, 8 in 2024-03-29T05:03:08.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:08 smithi187 ceph-mon[93536]: failed to encode map e630 with expected crc 2024-03-29T05:03:08.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:08 smithi187 ceph-mon[93536]: failed to encode map e630 with expected crc 2024-03-29T05:03:09.313 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:03:08] "GET /metrics HTTP/1.1" 200 50885 "" "Prometheus/2.43.0" 2024-03-29T05:03:09.579 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:09 smithi181 ceph-mon[106466]: failed to encode map e630 with expected crc 2024-03-29T05:03:09.579 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:09 smithi181 ceph-mon[106466]: osdmap e631: 8 total, 8 up, 8 in 2024-03-29T05:03:09.580 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:09 smithi181 ceph-mon[106466]: failed to encode map e631 with expected crc 2024-03-29T05:03:09.580 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:09 smithi181 ceph-mon[106466]: pgmap v620: 361 pgs: 17 creating+peering, 344 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 100 KiB/s rd, 8.2 KiB/s wr, 144 op/s 2024-03-29T05:03:09.580 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:09 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-29T05:03:09.580 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:09 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-29T05:03:09.580 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:09 smithi181 ceph-mon[101970]: failed to encode map e630 with expected crc 2024-03-29T05:03:09.580 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:09 smithi181 ceph-mon[101970]: osdmap e631: 8 total, 8 up, 8 in 2024-03-29T05:03:09.580 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:09 smithi181 ceph-mon[101970]: failed to encode map e631 with expected crc 2024-03-29T05:03:09.580 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:09 smithi181 ceph-mon[101970]: pgmap v620: 361 pgs: 17 creating+peering, 344 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 100 KiB/s rd, 8.2 KiB/s wr, 144 op/s 2024-03-29T05:03:09.580 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:09 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-29T05:03:09.580 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:09 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-29T05:03:09.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:09 smithi187 ceph-mon[93536]: failed to encode map e630 with expected crc 2024-03-29T05:03:09.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:09 smithi187 ceph-mon[93536]: osdmap e631: 8 total, 8 up, 8 in 2024-03-29T05:03:09.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:09 smithi187 ceph-mon[93536]: failed to encode map e631 with expected crc 2024-03-29T05:03:09.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:09 smithi187 ceph-mon[93536]: pgmap v620: 361 pgs: 17 creating+peering, 344 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 100 KiB/s rd, 8.2 KiB/s wr, 144 op/s 2024-03-29T05:03:09.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:09 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-29T05:03:09.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:09 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-29T05:03:10.691 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:10 smithi181 ceph-mon[101970]: failed to encode map e631 with expected crc 2024-03-29T05:03:10.691 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:10 smithi181 ceph-mon[101970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:03:10.691 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:10 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-29T05:03:10.691 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:10 smithi181 ceph-mon[101970]: osdmap e632: 8 total, 8 up, 8 in 2024-03-29T05:03:10.691 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:10 smithi181 ceph-mon[101970]: failed to encode map e632 with expected crc 2024-03-29T05:03:10.691 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:10 smithi181 ceph-mon[106466]: failed to encode map e631 with expected crc 2024-03-29T05:03:10.691 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:10 smithi181 ceph-mon[106466]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:03:10.691 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:10 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-29T05:03:10.692 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:10 smithi181 ceph-mon[106466]: osdmap e632: 8 total, 8 up, 8 in 2024-03-29T05:03:10.692 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:10 smithi181 ceph-mon[106466]: failed to encode map e632 with expected crc 2024-03-29T05:03:10.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:10 smithi187 ceph-mon[93536]: failed to encode map e631 with expected crc 2024-03-29T05:03:10.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:10 smithi187 ceph-mon[93536]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:03:10.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:10 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-29T05:03:10.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:10 smithi187 ceph-mon[93536]: osdmap e632: 8 total, 8 up, 8 in 2024-03-29T05:03:10.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:10 smithi187 ceph-mon[93536]: failed to encode map e632 with expected crc 2024-03-29T05:03:11.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:11 smithi181 ceph-mon[101970]: failed to encode map e632 with expected crc 2024-03-29T05:03:11.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:11 smithi181 ceph-mon[101970]: failed to encode map e632 with expected crc 2024-03-29T05:03:11.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:11 smithi181 ceph-mon[101970]: failed to encode map e632 with expected crc 2024-03-29T05:03:11.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:11 smithi181 ceph-mon[101970]: pgmap v622: 361 pgs: 17 creating+peering, 344 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 5.7 KiB/s wr, 80 op/s 2024-03-29T05:03:11.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:11.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:11.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:03:11.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:11 smithi181 ceph-mon[106466]: failed to encode map e632 with expected crc 2024-03-29T05:03:11.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:11 smithi181 ceph-mon[106466]: failed to encode map e632 with expected crc 2024-03-29T05:03:11.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:11 smithi181 ceph-mon[106466]: failed to encode map e632 with expected crc 2024-03-29T05:03:11.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:11 smithi181 ceph-mon[106466]: pgmap v622: 361 pgs: 17 creating+peering, 344 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 5.7 KiB/s wr, 80 op/s 2024-03-29T05:03:11.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:11.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:11.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:03:11.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:11 smithi187 ceph-mon[93536]: failed to encode map e632 with expected crc 2024-03-29T05:03:11.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:11 smithi187 ceph-mon[93536]: failed to encode map e632 with expected crc 2024-03-29T05:03:11.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:11 smithi187 ceph-mon[93536]: failed to encode map e632 with expected crc 2024-03-29T05:03:11.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:11 smithi187 ceph-mon[93536]: pgmap v622: 361 pgs: 17 creating+peering, 344 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 5.7 KiB/s wr, 80 op/s 2024-03-29T05:03:11.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:11.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:11.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:03:12.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:03:12.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:03:12.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:12 smithi181 ceph-mon[101970]: pgmap v623: 361 pgs: 11 creating+peering, 350 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 424 KiB/s wr, 51 op/s 2024-03-29T05:03:12.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:12 smithi181 ceph-mon[101970]: pgmap v624: 361 pgs: 11 creating+peering, 350 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 526 KiB/s wr, 18 op/s 2024-03-29T05:03:12.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:12.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:03:12.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:03:12.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:12 smithi181 ceph-mon[106466]: pgmap v623: 361 pgs: 11 creating+peering, 350 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 424 KiB/s wr, 51 op/s 2024-03-29T05:03:12.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:12 smithi181 ceph-mon[106466]: pgmap v624: 361 pgs: 11 creating+peering, 350 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 526 KiB/s wr, 18 op/s 2024-03-29T05:03:12.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:12.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:03:12.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:03:12.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:12 smithi187 ceph-mon[93536]: pgmap v623: 361 pgs: 11 creating+peering, 350 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 424 KiB/s wr, 51 op/s 2024-03-29T05:03:12.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:12 smithi187 ceph-mon[93536]: pgmap v624: 361 pgs: 11 creating+peering, 350 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 526 KiB/s wr, 18 op/s 2024-03-29T05:03:12.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:13.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:13 smithi187 ceph-mon[93536]: failed to encode map e633 with expected crc 2024-03-29T05:03:13.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:13 smithi187 ceph-mon[93536]: osdmap e633: 8 total, 8 up, 8 in 2024-03-29T05:03:13.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:13 smithi187 ceph-mon[93536]: failed to encode map e633 with expected crc 2024-03-29T05:03:13.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:13 smithi187 ceph-mon[93536]: failed to encode map e633 with expected crc 2024-03-29T05:03:13.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:13 smithi181 ceph-mon[101970]: failed to encode map e633 with expected crc 2024-03-29T05:03:13.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:13 smithi181 ceph-mon[101970]: osdmap e633: 8 total, 8 up, 8 in 2024-03-29T05:03:13.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:13 smithi181 ceph-mon[101970]: failed to encode map e633 with expected crc 2024-03-29T05:03:13.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:13 smithi181 ceph-mon[101970]: failed to encode map e633 with expected crc 2024-03-29T05:03:13.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:13 smithi181 ceph-mon[106466]: failed to encode map e633 with expected crc 2024-03-29T05:03:13.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:13 smithi181 ceph-mon[106466]: osdmap e633: 8 total, 8 up, 8 in 2024-03-29T05:03:13.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:13 smithi181 ceph-mon[106466]: failed to encode map e633 with expected crc 2024-03-29T05:03:13.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:13 smithi181 ceph-mon[106466]: failed to encode map e633 with expected crc 2024-03-29T05:03:14.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:14 smithi187 ceph-mon[93536]: pgmap v626: 361 pgs: 361 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 422 KiB/s wr, 41 op/s 2024-03-29T05:03:14.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:14 smithi181 ceph-mon[101970]: pgmap v626: 361 pgs: 361 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 422 KiB/s wr, 41 op/s 2024-03-29T05:03:14.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:14 smithi181 ceph-mon[106466]: pgmap v626: 361 pgs: 361 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 422 KiB/s wr, 41 op/s 2024-03-29T05:03:16.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:16 smithi187 ceph-mon[93536]: pgmap v627: 361 pgs: 361 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 350 KiB/s wr, 34 op/s 2024-03-29T05:03:16.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:16 smithi181 ceph-mon[101970]: pgmap v627: 361 pgs: 361 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 350 KiB/s wr, 34 op/s 2024-03-29T05:03:16.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:16 smithi181 ceph-mon[106466]: pgmap v627: 361 pgs: 361 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 350 KiB/s wr, 34 op/s 2024-03-29T05:03:18.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:18 smithi187 ceph-mon[93536]: pgmap v628: 361 pgs: 361 active+clean; 8.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.2 MiB/s wr, 31 op/s 2024-03-29T05:03:18.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:18 smithi187 ceph-mon[93536]: osdmap e634: 8 total, 8 up, 8 in 2024-03-29T05:03:18.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:18 smithi187 ceph-mon[93536]: failed to encode map e634 with expected crc 2024-03-29T05:03:18.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:18 smithi187 ceph-mon[93536]: failed to encode map e634 with expected crc 2024-03-29T05:03:18.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:18 smithi181 ceph-mon[101970]: pgmap v628: 361 pgs: 361 active+clean; 8.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.2 MiB/s wr, 31 op/s 2024-03-29T05:03:18.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:18 smithi181 ceph-mon[101970]: osdmap e634: 8 total, 8 up, 8 in 2024-03-29T05:03:18.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:18 smithi181 ceph-mon[101970]: failed to encode map e634 with expected crc 2024-03-29T05:03:18.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:18 smithi181 ceph-mon[101970]: failed to encode map e634 with expected crc 2024-03-29T05:03:18.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:18 smithi181 ceph-mon[106466]: pgmap v628: 361 pgs: 361 active+clean; 8.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.2 MiB/s wr, 31 op/s 2024-03-29T05:03:18.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:18 smithi181 ceph-mon[106466]: osdmap e634: 8 total, 8 up, 8 in 2024-03-29T05:03:18.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:18 smithi181 ceph-mon[106466]: failed to encode map e634 with expected crc 2024-03-29T05:03:18.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:18 smithi181 ceph-mon[106466]: failed to encode map e634 with expected crc 2024-03-29T05:03:19.455 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:03:18] "GET /metrics HTTP/1.1" 200 55666 "" "Prometheus/2.43.0" 2024-03-29T05:03:20.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:20 smithi187 ceph-mon[93536]: pgmap v630: 361 pgs: 361 active+clean; 8.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 769 KiB/s wr, 23 op/s 2024-03-29T05:03:20.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:20 smithi181 ceph-mon[101970]: pgmap v630: 361 pgs: 361 active+clean; 8.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 769 KiB/s wr, 23 op/s 2024-03-29T05:03:20.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:20 smithi181 ceph-mon[106466]: pgmap v630: 361 pgs: 361 active+clean; 8.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 769 KiB/s wr, 23 op/s 2024-03-29T05:03:22.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:22.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:22 smithi187 ceph-mon[93536]: pgmap v631: 361 pgs: 361 active+clean; 8.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 574 B/s rd, 690 KiB/s wr, 1 op/s 2024-03-29T05:03:22.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:22.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:22 smithi181 ceph-mon[101970]: pgmap v631: 361 pgs: 361 active+clean; 8.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 574 B/s rd, 690 KiB/s wr, 1 op/s 2024-03-29T05:03:22.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:22.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:22 smithi181 ceph-mon[106466]: pgmap v631: 361 pgs: 361 active+clean; 8.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 574 B/s rd, 690 KiB/s wr, 1 op/s 2024-03-29T05:03:24.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:24 smithi187 ceph-mon[93536]: pgmap v632: 361 pgs: 361 active+clean; 13 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-03-29T05:03:24.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:24 smithi181 ceph-mon[101970]: pgmap v632: 361 pgs: 361 active+clean; 13 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-03-29T05:03:24.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:24 smithi181 ceph-mon[106466]: pgmap v632: 361 pgs: 361 active+clean; 13 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-03-29T05:03:26.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:26 smithi187 ceph-mon[93536]: pgmap v633: 361 pgs: 361 active+clean; 13 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-03-29T05:03:26.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:03:26.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:26 smithi181 ceph-mon[106466]: pgmap v633: 361 pgs: 361 active+clean; 13 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-03-29T05:03:26.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:03:26.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:26 smithi181 ceph-mon[101970]: pgmap v633: 361 pgs: 361 active+clean; 13 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-03-29T05:03:26.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:03:27.821 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.RemoveFullTry (22935 ms) 2024-03-29T05:03:27.821 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.RemoveFullTryDataPool 2024-03-29T05:03:27.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:27 smithi187 ceph-mon[93536]: pool 'test-librbd-smithi181-86707-23' is full (reached quota's max_bytes: 10 MiB) 2024-03-29T05:03:27.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:27 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-29T05:03:27.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:27 smithi187 ceph-mon[93536]: osdmap e635: 8 total, 8 up, 8 in 2024-03-29T05:03:27.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:27 smithi181 ceph-mon[101970]: pool 'test-librbd-smithi181-86707-23' is full (reached quota's max_bytes: 10 MiB) 2024-03-29T05:03:27.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:27 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-29T05:03:27.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:27 smithi181 ceph-mon[101970]: osdmap e635: 8 total, 8 up, 8 in 2024-03-29T05:03:27.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:27 smithi181 ceph-mon[106466]: pool 'test-librbd-smithi181-86707-23' is full (reached quota's max_bytes: 10 MiB) 2024-03-29T05:03:27.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:27 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-29T05:03:27.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:27 smithi181 ceph-mon[106466]: osdmap e635: 8 total, 8 up, 8 in 2024-03-29T05:03:28.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:28 smithi187 ceph-mon[93536]: pgmap v634: 361 pgs: 361 active+clean; 18 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 922 KiB/s wr, 3 op/s 2024-03-29T05:03:28.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:28 smithi187 ceph-mon[93536]: failed to encode map e635 with expected crc 2024-03-29T05:03:28.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:28 smithi187 ceph-mon[93536]: failed to encode map e635 with expected crc 2024-03-29T05:03:28.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:28 smithi181 ceph-mon[101970]: pgmap v634: 361 pgs: 361 active+clean; 18 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 922 KiB/s wr, 3 op/s 2024-03-29T05:03:28.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:28 smithi181 ceph-mon[101970]: failed to encode map e635 with expected crc 2024-03-29T05:03:28.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:28 smithi181 ceph-mon[101970]: failed to encode map e635 with expected crc 2024-03-29T05:03:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:28 smithi181 ceph-mon[106466]: pgmap v634: 361 pgs: 361 active+clean; 18 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 922 KiB/s wr, 3 op/s 2024-03-29T05:03:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:28 smithi181 ceph-mon[106466]: failed to encode map e635 with expected crc 2024-03-29T05:03:28.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:28 smithi181 ceph-mon[106466]: failed to encode map e635 with expected crc 2024-03-29T05:03:29.455 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:03:28] "GET /metrics HTTP/1.1" 200 55720 "" "Prometheus/2.43.0" 2024-03-29T05:03:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[106466]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[106466]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[106466]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[106466]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[106466]: osdmap e636: 8 total, 8 up, 8 in 2024-03-29T05:03:29.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[106466]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[106466]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[106466]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[106466]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[106466]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3106620027' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:29.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:29.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[101970]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[101970]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[101970]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[101970]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[101970]: osdmap e636: 8 total, 8 up, 8 in 2024-03-29T05:03:29.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[101970]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[101970]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[101970]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[101970]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[101970]: failed to encode map e636 with expected crc 2024-03-29T05:03:29.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3106620027' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:29.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:29 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:30.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:29 smithi187 ceph-mon[93536]: failed to encode map e636 with expected crc 2024-03-29T05:03:30.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:29 smithi187 ceph-mon[93536]: failed to encode map e636 with expected crc 2024-03-29T05:03:30.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:29 smithi187 ceph-mon[93536]: failed to encode map e636 with expected crc 2024-03-29T05:03:30.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:29 smithi187 ceph-mon[93536]: failed to encode map e636 with expected crc 2024-03-29T05:03:30.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:29 smithi187 ceph-mon[93536]: osdmap e636: 8 total, 8 up, 8 in 2024-03-29T05:03:30.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:29 smithi187 ceph-mon[93536]: failed to encode map e636 with expected crc 2024-03-29T05:03:30.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:29 smithi187 ceph-mon[93536]: failed to encode map e636 with expected crc 2024-03-29T05:03:30.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:29 smithi187 ceph-mon[93536]: failed to encode map e636 with expected crc 2024-03-29T05:03:30.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:29 smithi187 ceph-mon[93536]: failed to encode map e636 with expected crc 2024-03-29T05:03:30.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:29 smithi187 ceph-mon[93536]: failed to encode map e636 with expected crc 2024-03-29T05:03:30.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:29 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3106620027' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:30.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:29 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:30 smithi181 ceph-mon[106466]: pgmap v637: 393 pgs: 32 unknown, 361 active+clean; 18 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.1 MiB/s wr, 4 op/s 2024-03-29T05:03:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:30 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:03:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:30 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:03:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:30 smithi181 ceph-mon[106466]: failed to encode map e637 with expected crc 2024-03-29T05:03:30.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:30 smithi181 ceph-mon[106466]: failed to encode map e637 with expected crc 2024-03-29T05:03:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:30 smithi181 ceph-mon[106466]: failed to encode map e637 with expected crc 2024-03-29T05:03:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:30 smithi181 ceph-mon[106466]: failed to encode map e637 with expected crc 2024-03-29T05:03:30.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:30 smithi181 ceph-mon[106466]: osdmap e637: 8 total, 8 up, 8 in 2024-03-29T05:03:30.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:30 smithi181 ceph-mon[101970]: pgmap v637: 393 pgs: 32 unknown, 361 active+clean; 18 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.1 MiB/s wr, 4 op/s 2024-03-29T05:03:30.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:30 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:03:30.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:30 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:03:30.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:30 smithi181 ceph-mon[101970]: failed to encode map e637 with expected crc 2024-03-29T05:03:30.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:30 smithi181 ceph-mon[101970]: failed to encode map e637 with expected crc 2024-03-29T05:03:30.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:30 smithi181 ceph-mon[101970]: failed to encode map e637 with expected crc 2024-03-29T05:03:30.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:30 smithi181 ceph-mon[101970]: failed to encode map e637 with expected crc 2024-03-29T05:03:30.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:30 smithi181 ceph-mon[101970]: osdmap e637: 8 total, 8 up, 8 in 2024-03-29T05:03:31.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:30 smithi187 ceph-mon[93536]: pgmap v637: 393 pgs: 32 unknown, 361 active+clean; 18 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.1 MiB/s wr, 4 op/s 2024-03-29T05:03:31.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:30 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:03:31.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:30 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:03:31.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:30 smithi187 ceph-mon[93536]: failed to encode map e637 with expected crc 2024-03-29T05:03:31.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:30 smithi187 ceph-mon[93536]: failed to encode map e637 with expected crc 2024-03-29T05:03:31.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:30 smithi187 ceph-mon[93536]: failed to encode map e637 with expected crc 2024-03-29T05:03:31.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:30 smithi187 ceph-mon[93536]: failed to encode map e637 with expected crc 2024-03-29T05:03:31.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:30 smithi187 ceph-mon[93536]: osdmap e637: 8 total, 8 up, 8 in 2024-03-29T05:03:31.697 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:03:31.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[106466]: failed to encode map e638 with expected crc 2024-03-29T05:03:31.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[106466]: failed to encode map e638 with expected crc 2024-03-29T05:03:31.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[106466]: failed to encode map e638 with expected crc 2024-03-29T05:03:31.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[106466]: osdmap e638: 8 total, 8 up, 8 in 2024-03-29T05:03:31.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[106466]: failed to encode map e638 with expected crc 2024-03-29T05:03:31.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[106466]: failed to encode map e638 with expected crc 2024-03-29T05:03:31.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[106466]: failed to encode map e638 with expected crc 2024-03-29T05:03:31.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[106466]: failed to encode map e638 with expected crc 2024-03-29T05:03:31.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2859042718' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:31.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:31.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[101970]: failed to encode map e638 with expected crc 2024-03-29T05:03:31.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[101970]: failed to encode map e638 with expected crc 2024-03-29T05:03:31.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[101970]: failed to encode map e638 with expected crc 2024-03-29T05:03:31.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[101970]: osdmap e638: 8 total, 8 up, 8 in 2024-03-29T05:03:31.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[101970]: failed to encode map e638 with expected crc 2024-03-29T05:03:31.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[101970]: failed to encode map e638 with expected crc 2024-03-29T05:03:31.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[101970]: failed to encode map e638 with expected crc 2024-03-29T05:03:31.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[101970]: failed to encode map e638 with expected crc 2024-03-29T05:03:31.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2859042718' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:31.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:31.957 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:03:31] ENGINE Bus STOPPING 2024-03-29T05:03:31.957 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:03:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T05:03:31.957 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:03:31] ENGINE Bus STOPPED 2024-03-29T05:03:31.957 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:03:31] ENGINE Bus STARTING 2024-03-29T05:03:31.957 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:03:31] ENGINE Serving on http://:::9283 2024-03-29T05:03:31.957 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:31 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:03:31] ENGINE Bus STARTED 2024-03-29T05:03:32.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:31 smithi187 ceph-mon[93536]: failed to encode map e638 with expected crc 2024-03-29T05:03:32.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:31 smithi187 ceph-mon[93536]: failed to encode map e638 with expected crc 2024-03-29T05:03:32.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:31 smithi187 ceph-mon[93536]: failed to encode map e638 with expected crc 2024-03-29T05:03:32.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:31 smithi187 ceph-mon[93536]: osdmap e638: 8 total, 8 up, 8 in 2024-03-29T05:03:32.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:31 smithi187 ceph-mon[93536]: failed to encode map e638 with expected crc 2024-03-29T05:03:32.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:31 smithi187 ceph-mon[93536]: failed to encode map e638 with expected crc 2024-03-29T05:03:32.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:31 smithi187 ceph-mon[93536]: failed to encode map e638 with expected crc 2024-03-29T05:03:32.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:31 smithi187 ceph-mon[93536]: failed to encode map e638 with expected crc 2024-03-29T05:03:32.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:31 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2859042718' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:32.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:31 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:03:32.831 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:32 smithi187 ceph-mon[93536]: pgmap v640: 425 pgs: 6 creating+peering, 49 unknown, 370 active+clean; 14 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.2 KiB/s rd, 0 B/s wr, 2 op/s 2024-03-29T05:03:32.831 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:32 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:03:32.831 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:32 smithi187 ceph-mon[93536]: failed to encode map e639 with expected crc 2024-03-29T05:03:32.831 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:32 smithi187 ceph-mon[93536]: failed to encode map e639 with expected crc 2024-03-29T05:03:32.831 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:32 smithi187 ceph-mon[93536]: failed to encode map e639 with expected crc 2024-03-29T05:03:32.831 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:32 smithi187 ceph-mon[93536]: failed to encode map e639 with expected crc 2024-03-29T05:03:32.831 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:32 smithi187 ceph-mon[93536]: osdmap e639: 8 total, 8 up, 8 in 2024-03-29T05:03:32.832 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:32 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' 2024-03-29T05:03:32.832 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:32 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:03:32.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[106466]: pgmap v640: 425 pgs: 6 creating+peering, 49 unknown, 370 active+clean; 14 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.2 KiB/s rd, 0 B/s wr, 2 op/s 2024-03-29T05:03:32.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:03:32.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[106466]: failed to encode map e639 with expected crc 2024-03-29T05:03:32.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[106466]: failed to encode map e639 with expected crc 2024-03-29T05:03:32.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[106466]: failed to encode map e639 with expected crc 2024-03-29T05:03:32.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[106466]: failed to encode map e639 with expected crc 2024-03-29T05:03:32.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[106466]: osdmap e639: 8 total, 8 up, 8 in 2024-03-29T05:03:32.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' 2024-03-29T05:03:32.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:03:32.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[101970]: pgmap v640: 425 pgs: 6 creating+peering, 49 unknown, 370 active+clean; 14 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.2 KiB/s rd, 0 B/s wr, 2 op/s 2024-03-29T05:03:32.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:03:32.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[101970]: failed to encode map e639 with expected crc 2024-03-29T05:03:32.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[101970]: failed to encode map e639 with expected crc 2024-03-29T05:03:32.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[101970]: failed to encode map e639 with expected crc 2024-03-29T05:03:32.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[101970]: failed to encode map e639 with expected crc 2024-03-29T05:03:32.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[101970]: osdmap e639: 8 total, 8 up, 8 in 2024-03-29T05:03:32.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' 2024-03-29T05:03:32.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:32 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:03:33.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:33 smithi187 ceph-mon[93536]: failed to encode map e640 with expected crc 2024-03-29T05:03:33.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:33 smithi187 ceph-mon[93536]: failed to encode map e640 with expected crc 2024-03-29T05:03:33.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:33 smithi187 ceph-mon[93536]: failed to encode map e640 with expected crc 2024-03-29T05:03:33.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:33 smithi187 ceph-mon[93536]: failed to encode map e640 with expected crc 2024-03-29T05:03:33.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:33 smithi187 ceph-mon[93536]: osdmap e640: 8 total, 8 up, 8 in 2024-03-29T05:03:33.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:33 smithi181 ceph-mon[106466]: failed to encode map e640 with expected crc 2024-03-29T05:03:33.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:33 smithi181 ceph-mon[106466]: failed to encode map e640 with expected crc 2024-03-29T05:03:33.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:33 smithi181 ceph-mon[106466]: failed to encode map e640 with expected crc 2024-03-29T05:03:33.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:33 smithi181 ceph-mon[106466]: failed to encode map e640 with expected crc 2024-03-29T05:03:33.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:33 smithi181 ceph-mon[106466]: osdmap e640: 8 total, 8 up, 8 in 2024-03-29T05:03:33.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:33 smithi181 ceph-mon[101970]: failed to encode map e640 with expected crc 2024-03-29T05:03:33.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:33 smithi181 ceph-mon[101970]: failed to encode map e640 with expected crc 2024-03-29T05:03:33.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:33 smithi181 ceph-mon[101970]: failed to encode map e640 with expected crc 2024-03-29T05:03:33.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:33 smithi181 ceph-mon[101970]: failed to encode map e640 with expected crc 2024-03-29T05:03:33.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:33 smithi181 ceph-mon[101970]: osdmap e640: 8 total, 8 up, 8 in 2024-03-29T05:03:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[106466]: pgmap v643: 425 pgs: 11 creating+peering, 414 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 258 KiB/s wr, 40 op/s 2024-03-29T05:03:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[106466]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:03:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[106466]: failed to encode map e641 with expected crc 2024-03-29T05:03:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[106466]: failed to encode map e641 with expected crc 2024-03-29T05:03:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[106466]: failed to encode map e641 with expected crc 2024-03-29T05:03:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[106466]: failed to encode map e641 with expected crc 2024-03-29T05:03:34.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[106466]: osdmap e641: 8 total, 8 up, 8 in 2024-03-29T05:03:34.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-29T05:03:34.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-29T05:03:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[101970]: pgmap v643: 425 pgs: 11 creating+peering, 414 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 258 KiB/s wr, 40 op/s 2024-03-29T05:03:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[101970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:03:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[101970]: failed to encode map e641 with expected crc 2024-03-29T05:03:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[101970]: failed to encode map e641 with expected crc 2024-03-29T05:03:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[101970]: failed to encode map e641 with expected crc 2024-03-29T05:03:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[101970]: failed to encode map e641 with expected crc 2024-03-29T05:03:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[101970]: osdmap e641: 8 total, 8 up, 8 in 2024-03-29T05:03:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-29T05:03:34.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:34 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-29T05:03:35.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:34 smithi187 ceph-mon[93536]: pgmap v643: 425 pgs: 11 creating+peering, 414 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 258 KiB/s wr, 40 op/s 2024-03-29T05:03:35.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:34 smithi187 ceph-mon[93536]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:03:35.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:34 smithi187 ceph-mon[93536]: failed to encode map e641 with expected crc 2024-03-29T05:03:35.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:34 smithi187 ceph-mon[93536]: failed to encode map e641 with expected crc 2024-03-29T05:03:35.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:34 smithi187 ceph-mon[93536]: failed to encode map e641 with expected crc 2024-03-29T05:03:35.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:34 smithi187 ceph-mon[93536]: failed to encode map e641 with expected crc 2024-03-29T05:03:35.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:34 smithi187 ceph-mon[93536]: osdmap e641: 8 total, 8 up, 8 in 2024-03-29T05:03:35.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:34 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-29T05:03:35.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:34 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-29T05:03:35.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:35 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-29T05:03:35.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:35 smithi181 ceph-mon[101970]: failed to encode map e642 with expected crc 2024-03-29T05:03:35.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:35 smithi181 ceph-mon[101970]: failed to encode map e642 with expected crc 2024-03-29T05:03:35.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:35 smithi181 ceph-mon[101970]: failed to encode map e642 with expected crc 2024-03-29T05:03:35.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:35 smithi181 ceph-mon[101970]: failed to encode map e642 with expected crc 2024-03-29T05:03:35.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:35 smithi181 ceph-mon[101970]: osdmap e642: 8 total, 8 up, 8 in 2024-03-29T05:03:35.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:35 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-29T05:03:35.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:35 smithi181 ceph-mon[106466]: failed to encode map e642 with expected crc 2024-03-29T05:03:35.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:35 smithi181 ceph-mon[106466]: failed to encode map e642 with expected crc 2024-03-29T05:03:35.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:35 smithi181 ceph-mon[106466]: failed to encode map e642 with expected crc 2024-03-29T05:03:35.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:35 smithi181 ceph-mon[106466]: failed to encode map e642 with expected crc 2024-03-29T05:03:35.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:35 smithi181 ceph-mon[106466]: osdmap e642: 8 total, 8 up, 8 in 2024-03-29T05:03:36.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:35 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi181-86707-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-29T05:03:36.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:35 smithi187 ceph-mon[93536]: failed to encode map e642 with expected crc 2024-03-29T05:03:36.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:35 smithi187 ceph-mon[93536]: failed to encode map e642 with expected crc 2024-03-29T05:03:36.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:35 smithi187 ceph-mon[93536]: failed to encode map e642 with expected crc 2024-03-29T05:03:36.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:35 smithi187 ceph-mon[93536]: failed to encode map e642 with expected crc 2024-03-29T05:03:36.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:35 smithi187 ceph-mon[93536]: osdmap e642: 8 total, 8 up, 8 in 2024-03-29T05:03:37.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:36 smithi187 ceph-mon[93536]: pgmap v646: 425 pgs: 11 creating+peering, 414 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 258 KiB/s wr, 37 op/s 2024-03-29T05:03:37.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:37.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:37.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:03:37.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:03:37.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:37.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:36 smithi181 ceph-mon[106466]: pgmap v646: 425 pgs: 11 creating+peering, 414 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 258 KiB/s wr, 37 op/s 2024-03-29T05:03:37.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:37.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:37.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:03:37.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:03:37.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:37.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:36 smithi181 ceph-mon[101970]: pgmap v646: 425 pgs: 11 creating+peering, 414 active+clean; 584 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 258 KiB/s wr, 37 op/s 2024-03-29T05:03:37.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:37.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:37.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:03:37.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:03:37.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:38.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:37 smithi187 ceph-mon[93536]: pgmap v647: 425 pgs: 5 creating+peering, 420 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 420 KiB/s wr, 23 op/s 2024-03-29T05:03:38.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:37 smithi187 ceph-mon[93536]: pgmap v648: 425 pgs: 5 creating+peering, 420 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 263 B/s rd, 528 KiB/s wr, 2 op/s 2024-03-29T05:03:38.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:37 smithi187 ceph-mon[93536]: pool 'test-librbd-smithi181-86707-23' no longer out of quota; removing NO_QUOTA flag 2024-03-29T05:03:38.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:37 smithi187 ceph-mon[93536]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-29T05:03:38.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:37 smithi187 ceph-mon[93536]: osdmap e643: 8 total, 8 up, 8 in 2024-03-29T05:03:38.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:37 smithi181 ceph-mon[101970]: pgmap v647: 425 pgs: 5 creating+peering, 420 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 420 KiB/s wr, 23 op/s 2024-03-29T05:03:38.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:37 smithi181 ceph-mon[101970]: pgmap v648: 425 pgs: 5 creating+peering, 420 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 263 B/s rd, 528 KiB/s wr, 2 op/s 2024-03-29T05:03:38.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:37 smithi181 ceph-mon[101970]: pool 'test-librbd-smithi181-86707-23' no longer out of quota; removing NO_QUOTA flag 2024-03-29T05:03:38.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:37 smithi181 ceph-mon[101970]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-29T05:03:38.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:37 smithi181 ceph-mon[101970]: osdmap e643: 8 total, 8 up, 8 in 2024-03-29T05:03:38.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:37 smithi181 ceph-mon[106466]: pgmap v647: 425 pgs: 5 creating+peering, 420 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 420 KiB/s wr, 23 op/s 2024-03-29T05:03:38.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:37 smithi181 ceph-mon[106466]: pgmap v648: 425 pgs: 5 creating+peering, 420 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 263 B/s rd, 528 KiB/s wr, 2 op/s 2024-03-29T05:03:38.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:37 smithi181 ceph-mon[106466]: pool 'test-librbd-smithi181-86707-23' no longer out of quota; removing NO_QUOTA flag 2024-03-29T05:03:38.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:37 smithi181 ceph-mon[106466]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-29T05:03:38.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:37 smithi181 ceph-mon[106466]: osdmap e643: 8 total, 8 up, 8 in 2024-03-29T05:03:39.356 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:03:38] "GET /metrics HTTP/1.1" 200 55720 "" "Prometheus/2.43.0" 2024-03-29T05:03:39.357 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:38 smithi181 ceph-mon[106466]: failed to encode map e643 with expected crc 2024-03-29T05:03:39.357 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:38 smithi181 ceph-mon[106466]: failed to encode map e643 with expected crc 2024-03-29T05:03:39.357 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:38 smithi181 ceph-mon[106466]: failed to encode map e643 with expected crc 2024-03-29T05:03:39.357 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:38 smithi181 ceph-mon[106466]: failed to encode map e643 with expected crc 2024-03-29T05:03:39.357 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:38 smithi181 ceph-mon[101970]: failed to encode map e643 with expected crc 2024-03-29T05:03:39.357 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:38 smithi181 ceph-mon[101970]: failed to encode map e643 with expected crc 2024-03-29T05:03:39.357 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:38 smithi181 ceph-mon[101970]: failed to encode map e643 with expected crc 2024-03-29T05:03:39.357 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:38 smithi181 ceph-mon[101970]: failed to encode map e643 with expected crc 2024-03-29T05:03:39.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:38 smithi187 ceph-mon[93536]: failed to encode map e643 with expected crc 2024-03-29T05:03:39.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:38 smithi187 ceph-mon[93536]: failed to encode map e643 with expected crc 2024-03-29T05:03:39.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:38 smithi187 ceph-mon[93536]: failed to encode map e643 with expected crc 2024-03-29T05:03:39.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:38 smithi187 ceph-mon[93536]: failed to encode map e643 with expected crc 2024-03-29T05:03:40.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:39 smithi187 ceph-mon[93536]: pgmap v650: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.6 KiB/s rd, 423 KiB/s wr, 11 op/s 2024-03-29T05:03:40.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:39 smithi181 ceph-mon[101970]: pgmap v650: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.6 KiB/s rd, 423 KiB/s wr, 11 op/s 2024-03-29T05:03:40.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:39 smithi181 ceph-mon[106466]: pgmap v650: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.6 KiB/s rd, 423 KiB/s wr, 11 op/s 2024-03-29T05:03:42.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:41 smithi187 ceph-mon[93536]: pgmap v651: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.5 KiB/s rd, 351 KiB/s wr, 9 op/s 2024-03-29T05:03:42.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:03:42.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:41 smithi181 ceph-mon[101970]: pgmap v651: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.5 KiB/s rd, 351 KiB/s wr, 9 op/s 2024-03-29T05:03:42.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:03:42.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:41 smithi181 ceph-mon[106466]: pgmap v651: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.5 KiB/s rd, 351 KiB/s wr, 9 op/s 2024-03-29T05:03:42.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:03:43.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:43 smithi187 ceph-mon[93536]: pgmap v652: 425 pgs: 425 active+clean; 8.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.3 KiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-03-29T05:03:43.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:43 smithi187 ceph-mon[93536]: failed to encode map e644 with expected crc 2024-03-29T05:03:43.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:43 smithi187 ceph-mon[93536]: failed to encode map e644 with expected crc 2024-03-29T05:03:43.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:43 smithi187 ceph-mon[93536]: failed to encode map e644 with expected crc 2024-03-29T05:03:43.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:43 smithi187 ceph-mon[93536]: failed to encode map e644 with expected crc 2024-03-29T05:03:43.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:43 smithi187 ceph-mon[93536]: osdmap e644: 8 total, 8 up, 8 in 2024-03-29T05:03:43.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:43 smithi181 ceph-mon[106466]: pgmap v652: 425 pgs: 425 active+clean; 8.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.3 KiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-03-29T05:03:43.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:43 smithi181 ceph-mon[106466]: failed to encode map e644 with expected crc 2024-03-29T05:03:43.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:43 smithi181 ceph-mon[106466]: failed to encode map e644 with expected crc 2024-03-29T05:03:43.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:43 smithi181 ceph-mon[106466]: failed to encode map e644 with expected crc 2024-03-29T05:03:43.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:43 smithi181 ceph-mon[106466]: failed to encode map e644 with expected crc 2024-03-29T05:03:43.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:43 smithi181 ceph-mon[106466]: osdmap e644: 8 total, 8 up, 8 in 2024-03-29T05:03:43.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:43 smithi181 ceph-mon[101970]: pgmap v652: 425 pgs: 425 active+clean; 8.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 5.3 KiB/s rd, 1.1 MiB/s wr, 10 op/s 2024-03-29T05:03:43.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:43 smithi181 ceph-mon[101970]: failed to encode map e644 with expected crc 2024-03-29T05:03:43.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:43 smithi181 ceph-mon[101970]: failed to encode map e644 with expected crc 2024-03-29T05:03:43.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:43 smithi181 ceph-mon[101970]: failed to encode map e644 with expected crc 2024-03-29T05:03:43.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:43 smithi181 ceph-mon[101970]: failed to encode map e644 with expected crc 2024-03-29T05:03:43.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:43 smithi181 ceph-mon[101970]: osdmap e644: 8 total, 8 up, 8 in 2024-03-29T05:03:45.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:45 smithi187 ceph-mon[93536]: pgmap v654: 425 pgs: 425 active+clean; 8.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.5 KiB/s rd, 769 KiB/s wr, 7 op/s 2024-03-29T05:03:45.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:45 smithi181 ceph-mon[101970]: pgmap v654: 425 pgs: 425 active+clean; 8.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.5 KiB/s rd, 769 KiB/s wr, 7 op/s 2024-03-29T05:03:45.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:45 smithi181 ceph-mon[106466]: pgmap v654: 425 pgs: 425 active+clean; 8.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.5 KiB/s rd, 769 KiB/s wr, 7 op/s 2024-03-29T05:03:47.871 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.RemoveFullTryDataPool (20050 ms) 2024-03-29T05:03:47.871 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestIO 2024-03-29T05:03:47.871 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:03:47.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:47 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:47.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:47 smithi187 ceph-mon[93536]: pgmap v655: 425 pgs: 425 active+clean; 13 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 802 B/s rd, 1.1 MiB/s wr, 2 op/s 2024-03-29T05:03:47.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:47 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:47.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:47 smithi181 ceph-mon[101970]: pgmap v655: 425 pgs: 425 active+clean; 13 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 802 B/s rd, 1.1 MiB/s wr, 2 op/s 2024-03-29T05:03:47.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:47 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:03:47.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:47 smithi181 ceph-mon[106466]: pgmap v655: 425 pgs: 425 active+clean; 13 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 802 B/s rd, 1.1 MiB/s wr, 2 op/s 2024-03-29T05:03:48.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:48 smithi187 ceph-mon[93536]: pool 'test-librbd-smithi181-86707-25' is full (reached quota's max_bytes: 10 MiB) 2024-03-29T05:03:48.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:48 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-29T05:03:48.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:48 smithi187 ceph-mon[93536]: osdmap e645: 8 total, 8 up, 8 in 2024-03-29T05:03:48.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:48 smithi187 ceph-mon[93536]: failed to encode map e645 with expected crc 2024-03-29T05:03:48.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:48 smithi187 ceph-mon[93536]: failed to encode map e645 with expected crc 2024-03-29T05:03:48.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:48 smithi181 ceph-mon[101970]: pool 'test-librbd-smithi181-86707-25' is full (reached quota's max_bytes: 10 MiB) 2024-03-29T05:03:48.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:48 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-29T05:03:48.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:48 smithi181 ceph-mon[101970]: osdmap e645: 8 total, 8 up, 8 in 2024-03-29T05:03:48.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:48 smithi181 ceph-mon[101970]: failed to encode map e645 with expected crc 2024-03-29T05:03:48.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:48 smithi181 ceph-mon[101970]: failed to encode map e645 with expected crc 2024-03-29T05:03:48.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:48 smithi181 ceph-mon[106466]: pool 'test-librbd-smithi181-86707-25' is full (reached quota's max_bytes: 10 MiB) 2024-03-29T05:03:48.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:48 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-29T05:03:48.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:48 smithi181 ceph-mon[106466]: osdmap e645: 8 total, 8 up, 8 in 2024-03-29T05:03:48.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:48 smithi181 ceph-mon[106466]: failed to encode map e645 with expected crc 2024-03-29T05:03:48.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:48 smithi181 ceph-mon[106466]: failed to encode map e645 with expected crc 2024-03-29T05:03:49.205 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:03:48] "GET /metrics HTTP/1.1" 200 60517 "" "Prometheus/2.43.0" 2024-03-29T05:03:49.424 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:03:49.424 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:03:49.424 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:03:49.424 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:03:49.425 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.426 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:03:49.426 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.426 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:49.426 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:03:49.426 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.426 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:03:49.426 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.426 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:49.426 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:03:49.426 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:03:49.426 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:03:49.426 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:03:49.426 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:03:49.426 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:03:49.427 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:03:49.427 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:03:49.427 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:03:49.427 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:03:49.427 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:03:49.427 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.427 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:03:49.427 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.427 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:49.427 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:03:49.427 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.427 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:03:49.427 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.427 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:49.427 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:03:49.427 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.428 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:03:49.428 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.428 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:49.428 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:03:49.428 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.428 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:03:49.428 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.428 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:49.428 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:03:49.428 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.428 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:03:49.428 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.428 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:49.428 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:03:49.428 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:03:49.429 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.430 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:03:49.430 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:03:49.430 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:03:49.430 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.430 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:03:49.430 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:03:49.430 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:03:49.430 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 512~512 = 512 2024-03-29T05:03:49.430 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.430 INFO:tasks.workunit.client.0.smithi181.stdout:aio discard: 1536~512 = 0 2024-03-29T05:03:49.430 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:49.430 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:49.430 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:49.430 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:49.430 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:49.430 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 0 2024-03-29T05:03:49.431 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.431 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.431 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 0 2024-03-29T05:03:49.431 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.431 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.431 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 16384 2024-03-29T05:03:49.431 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.431 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.431 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 16384 2024-03-29T05:03:49.431 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.431 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.431 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:49.431 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:49.431 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 49152 2024-03-29T05:03:49.431 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.431 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 49152 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 65536 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 65536 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 98304 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 98304 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.432 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.433 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 114688 2024-03-29T05:03:49.433 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.433 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.433 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 114688 2024-03-29T05:03:49.433 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.433 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.433 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:49.433 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:49.433 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 147456 2024-03-29T05:03:49.433 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.433 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.434 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 147456 2024-03-29T05:03:49.434 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.434 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.434 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 163840 2024-03-29T05:03:49.434 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.434 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.434 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 163840 2024-03-29T05:03:49.434 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.434 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.434 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:49.434 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:49.434 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 196608 2024-03-29T05:03:49.434 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.434 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.435 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 196608 2024-03-29T05:03:49.435 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.435 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.435 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 212992 2024-03-29T05:03:49.435 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.435 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.435 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 212992 2024-03-29T05:03:49.435 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.435 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.435 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:49.435 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:49.435 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.435 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:49.435 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.435 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:49.435 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:49.436 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.436 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.436 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.436 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:49.436 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.436 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:49.436 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:49.436 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.436 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.436 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.436 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:49.436 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.436 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:49.436 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:49.436 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.436 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.437 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.437 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:49.437 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.437 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:49.437 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:49.437 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.437 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.437 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.437 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:49.437 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:49.437 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.437 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:49.437 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:49.437 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.437 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:49.437 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.438 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:49.438 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:49.438 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.438 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.438 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.438 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:49.438 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:49.438 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:49.438 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:49.438 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:49.438 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:49.438 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:49.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:49 smithi187 ceph-mon[93536]: pgmap v657: 425 pgs: 425 active+clean; 14 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.4 MiB/s wr, 5 op/s 2024-03-29T05:03:49.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:49 smithi181 ceph-mon[101970]: pgmap v657: 425 pgs: 425 active+clean; 14 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.4 MiB/s wr, 5 op/s 2024-03-29T05:03:49.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:49 smithi181 ceph-mon[106466]: pgmap v657: 425 pgs: 425 active+clean; 14 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.4 MiB/s wr, 5 op/s 2024-03-29T05:03:50.756 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.756 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:50.756 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:50.756 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:50.756 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:50.756 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:50.756 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.756 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.756 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:50.757 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:50.757 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:50.757 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:50.757 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:50.757 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.757 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.757 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:50.757 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.757 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.757 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:50.757 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.757 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.757 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:50.757 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:50.757 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:50.758 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:50.758 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:50.758 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.758 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.758 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:50.758 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:50.758 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:50.758 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:50.758 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:50.758 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.758 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.758 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:50.758 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:50.758 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:50.758 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:50.759 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:50.759 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.759 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.759 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:50.759 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:50.759 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:50.759 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:50.759 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:50.759 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.759 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.759 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:50.759 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.759 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.759 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:50.759 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.760 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.760 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:50.760 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:50.760 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:50.760 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:50.760 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:50.760 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.760 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.760 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:50.760 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:50.760 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:50.760 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:50.760 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:50.760 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.760 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.761 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:50.761 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:50.761 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:50.761 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:50.761 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:50.761 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.761 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.761 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:50.761 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:50.761 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:50.761 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:50.761 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:50.761 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.761 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.761 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:50.761 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.762 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.762 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:50.762 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.762 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.762 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:50.762 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:50.762 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:50.762 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:50.762 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:50.762 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.762 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.762 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:50.762 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:50.762 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:50.762 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:50.763 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:50.763 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.763 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.763 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:50.763 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:50.763 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:50.763 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:50.763 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:50.763 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.763 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.763 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:50.763 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:50.763 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:50.763 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:50.763 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:50.764 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.764 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.764 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:50.764 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:50.764 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:50.764 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:50.764 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:03:50.764 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:03:50.764 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:03:50.764 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:03:50.764 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestIO (2885 ms) 2024-03-29T05:03:50.764 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS1 2024-03-29T05:03:50.764 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:03:51.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:51 smithi187 ceph-mon[93536]: pgmap v658: 425 pgs: 425 active+clean; 14 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 640 KiB/s wr, 3 op/s 2024-03-29T05:03:51.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:51 smithi181 ceph-mon[106466]: pgmap v658: 425 pgs: 425 active+clean; 14 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 640 KiB/s wr, 3 op/s 2024-03-29T05:03:51.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:51 smithi181 ceph-mon[101970]: pgmap v658: 425 pgs: 425 active+clean; 14 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 640 KiB/s wr, 3 op/s 2024-03-29T05:03:53.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:53 smithi187 ceph-mon[93536]: pgmap v659: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 440 KiB/s rd, 730 KiB/s wr, 874 op/s 2024-03-29T05:03:53.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:53 smithi187 ceph-mon[93536]: pool 'test-librbd-smithi181-86707-25' no longer out of quota; removing NO_QUOTA flag 2024-03-29T05:03:53.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:53 smithi187 ceph-mon[93536]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-29T05:03:53.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:53 smithi187 ceph-mon[93536]: failed to encode map e646 with expected crc 2024-03-29T05:03:53.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:53 smithi187 ceph-mon[93536]: failed to encode map e646 with expected crc 2024-03-29T05:03:53.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:53 smithi187 ceph-mon[93536]: osdmap e646: 8 total, 8 up, 8 in 2024-03-29T05:03:53.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:53 smithi181 ceph-mon[101970]: pgmap v659: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 440 KiB/s rd, 730 KiB/s wr, 874 op/s 2024-03-29T05:03:53.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:53 smithi181 ceph-mon[101970]: pool 'test-librbd-smithi181-86707-25' no longer out of quota; removing NO_QUOTA flag 2024-03-29T05:03:53.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:53 smithi181 ceph-mon[101970]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-29T05:03:53.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:53 smithi181 ceph-mon[101970]: failed to encode map e646 with expected crc 2024-03-29T05:03:53.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:53 smithi181 ceph-mon[101970]: failed to encode map e646 with expected crc 2024-03-29T05:03:53.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:53 smithi181 ceph-mon[101970]: osdmap e646: 8 total, 8 up, 8 in 2024-03-29T05:03:53.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:53 smithi181 ceph-mon[106466]: pgmap v659: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 440 KiB/s rd, 730 KiB/s wr, 874 op/s 2024-03-29T05:03:53.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:53 smithi181 ceph-mon[106466]: pool 'test-librbd-smithi181-86707-25' no longer out of quota; removing NO_QUOTA flag 2024-03-29T05:03:53.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:53 smithi181 ceph-mon[106466]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-29T05:03:53.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:53 smithi181 ceph-mon[106466]: failed to encode map e646 with expected crc 2024-03-29T05:03:53.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:53 smithi181 ceph-mon[106466]: failed to encode map e646 with expected crc 2024-03-29T05:03:53.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:53 smithi181 ceph-mon[106466]: osdmap e646: 8 total, 8 up, 8 in 2024-03-29T05:03:54.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:54 smithi187 ceph-mon[93536]: failed to encode map e646 with expected crc 2024-03-29T05:03:54.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:54 smithi181 ceph-mon[106466]: failed to encode map e646 with expected crc 2024-03-29T05:03:54.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:54 smithi181 ceph-mon[101970]: failed to encode map e646 with expected crc 2024-03-29T05:03:55.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:55 smithi187 ceph-mon[93536]: pgmap v661: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 546 KiB/s rd, 393 KiB/s wr, 1.08k op/s 2024-03-29T05:03:55.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:55 smithi181 ceph-mon[106466]: pgmap v661: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 546 KiB/s rd, 393 KiB/s wr, 1.08k op/s 2024-03-29T05:03:55.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:55 smithi181 ceph-mon[101970]: pgmap v661: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 546 KiB/s rd, 393 KiB/s wr, 1.08k op/s 2024-03-29T05:03:56.309 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:03:56.310 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:03:56.310 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:03:56.310 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:03:56.310 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:03:56.310 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.310 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:03:56.310 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.310 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:56.310 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:03:56.310 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.310 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:03:56.310 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.311 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:56.311 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:03:56.311 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.311 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:03:56.311 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.311 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:56.311 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:03:56.311 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.311 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:03:56.311 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.311 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:56.311 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:03:56.311 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.311 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:03:56.311 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.311 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:56.312 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:03:56.312 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:03:56.312 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:03:56.312 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:03:56.312 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:03:56.312 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:03:56.312 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:03:56.312 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:03:56.312 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:03:56.312 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:03:56.312 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:03:56.312 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.312 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:03:56.312 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.312 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:56.312 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:03:56.313 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.313 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:03:56.313 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.313 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:56.313 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:03:56.313 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.313 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:03:56.313 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.313 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:56.313 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:03:56.313 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.314 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:03:56.314 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.314 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:56.314 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:03:56.314 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.314 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:03:56.314 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.314 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:56.314 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:03:56.314 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:56.314 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:56.314 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:56.314 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:56.314 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:56.314 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.315 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:03:56.315 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:03:56.315 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:03:56.315 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.315 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:03:56.315 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:03:56.315 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:03:56.315 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.315 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:03:56.315 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:03:56.315 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:03:56.315 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.315 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:03:56.315 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:03:56.315 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:03:56.316 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.316 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:03:56.316 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:03:56.316 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:03:56.316 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 512~512 = 512 2024-03-29T05:03:56.316 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.316 INFO:tasks.workunit.client.0.smithi181.stdout:aio discard: 1536~512 = 0 2024-03-29T05:03:56.316 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:56.316 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:56.316 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:56.316 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:56.316 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:03:56.316 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 0 2024-03-29T05:03:56.316 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.317 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.317 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 0 2024-03-29T05:03:56.317 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.317 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.317 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 16384 2024-03-29T05:03:56.317 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.317 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.317 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 16384 2024-03-29T05:03:56.317 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.317 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.317 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:56.317 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:56.317 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 49152 2024-03-29T05:03:56.317 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.317 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.318 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 49152 2024-03-29T05:03:56.318 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.318 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.318 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 65536 2024-03-29T05:03:56.318 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.318 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.318 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 65536 2024-03-29T05:03:56.318 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.318 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.318 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:56.318 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:56.318 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 98304 2024-03-29T05:03:56.318 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.318 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.318 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 98304 2024-03-29T05:03:56.319 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.319 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.319 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 114688 2024-03-29T05:03:56.319 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.319 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.319 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 114688 2024-03-29T05:03:56.319 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.319 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.319 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:56.319 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:56.319 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 147456 2024-03-29T05:03:56.319 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.320 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.320 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 147456 2024-03-29T05:03:56.320 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.320 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.320 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 163840 2024-03-29T05:03:56.320 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.320 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.320 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 163840 2024-03-29T05:03:56.320 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.320 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.320 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:56.320 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:56.321 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 196608 2024-03-29T05:03:56.321 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.321 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.321 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 196608 2024-03-29T05:03:56.321 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.321 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.321 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 212992 2024-03-29T05:03:56.321 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.321 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.321 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 212992 2024-03-29T05:03:56.321 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.321 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.321 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:56.321 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:56.321 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.322 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:56.322 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.322 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:56.322 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:56.322 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.322 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.322 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.322 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:56.322 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.322 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:56.322 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:56.322 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.322 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.322 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.322 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:56.323 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.323 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:56.323 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:56.323 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.323 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.323 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.323 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:56.323 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.323 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:56.323 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:56.323 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.323 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.323 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.323 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:56.323 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:56.324 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.324 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:56.324 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:03:56.324 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.324 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:56.324 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.324 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:56.324 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:56.324 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.324 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.324 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.324 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:03:56.324 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:03:56.324 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:03:56.324 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:03:56.325 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:03:56.325 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:03:56.325 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:03:56.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:03:56.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:03:56.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:03:57.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:57 smithi187 ceph-mon[93536]: pgmap v662: 425 pgs: 425 active+clean; 6.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 489 KiB/s rd, 696 KiB/s wr, 970 op/s 2024-03-29T05:03:57.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:57 smithi181 ceph-mon[101970]: pgmap v662: 425 pgs: 425 active+clean; 6.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 489 KiB/s rd, 696 KiB/s wr, 970 op/s 2024-03-29T05:03:57.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:57 smithi181 ceph-mon[106466]: pgmap v662: 425 pgs: 425 active+clean; 6.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 489 KiB/s rd, 696 KiB/s wr, 970 op/s 2024-03-29T05:03:59.207 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:03:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:03:58] "GET /metrics HTTP/1.1" 200 60483 "" "Prometheus/2.43.0" 2024-03-29T05:03:59.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:03:59 smithi187 ceph-mon[93536]: pgmap v663: 425 pgs: 425 active+clean; 6.8 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 961 KiB/s wr, 1.52k op/s 2024-03-29T05:03:59.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:03:59 smithi181 ceph-mon[106466]: pgmap v663: 425 pgs: 425 active+clean; 6.8 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 961 KiB/s wr, 1.52k op/s 2024-03-29T05:03:59.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:03:59 smithi181 ceph-mon[101970]: pgmap v663: 425 pgs: 425 active+clean; 6.8 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 961 KiB/s wr, 1.52k op/s 2024-03-29T05:04:00.127 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.127 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:00.127 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:00.127 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:00.127 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:00.127 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:00.127 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.128 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.128 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:00.128 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:00.128 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:00.128 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:00.128 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:00.128 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.128 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.128 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:00.128 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.128 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.128 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:00.128 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.128 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.128 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:00.129 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:00.129 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:00.129 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:00.129 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:00.129 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.129 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.129 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:00.129 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:00.129 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:00.129 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:00.129 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:00.129 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.129 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.129 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:00.130 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:00.130 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:00.130 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:00.130 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:00.130 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.130 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.130 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:00.130 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:00.130 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:00.130 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:00.130 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:00.130 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.130 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.130 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:00.130 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.131 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.131 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:00.131 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.131 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.131 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:00.131 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:00.131 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:00.131 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:00.131 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:00.131 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.131 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.132 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:00.132 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:00.132 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:00.132 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:00.132 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:00.132 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.132 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.132 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:00.132 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:00.132 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:00.132 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:00.132 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:00.132 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.132 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.133 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:00.133 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:00.133 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:00.133 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:00.133 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:00.133 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.133 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.133 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:00.133 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.133 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.133 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:00.133 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.133 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.133 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:00.133 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:00.133 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:00.134 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:00.134 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:00.134 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.134 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.134 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:00.134 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:00.134 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:00.134 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:00.134 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:00.134 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.134 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.135 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:00.135 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:00.135 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:00.135 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:00.135 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:00.135 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.135 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.136 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:00.136 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:00.136 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:00.136 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:00.136 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:00.136 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.136 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.137 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:00.137 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:00.137 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:00.137 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:00.137 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:00.137 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:00.137 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:00.138 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:00.138 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 4 2024-03-29T05:04:00.138 INFO:tasks.workunit.client.0.smithi181.stdout:read: 4 2024-03-29T05:04:00.138 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS1 (9371 ms) 2024-03-29T05:04:00.138 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS2 2024-03-29T05:04:00.138 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:01.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:01 smithi187 ceph-mon[93536]: pgmap v664: 425 pgs: 425 active+clean; 6.8 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 961 KiB/s wr, 1.52k op/s 2024-03-29T05:04:01.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:01 smithi181 ceph-mon[101970]: pgmap v664: 425 pgs: 425 active+clean; 6.8 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 961 KiB/s wr, 1.52k op/s 2024-03-29T05:04:01.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:01 smithi181 ceph-mon[106466]: pgmap v664: 425 pgs: 425 active+clean; 6.8 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 961 KiB/s wr, 1.52k op/s 2024-03-29T05:04:03.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:03 smithi187 ceph-mon[93536]: pgmap v665: 425 pgs: 425 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 874 KiB/s rd, 831 KiB/s wr, 854 op/s 2024-03-29T05:04:03.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:03 smithi181 ceph-mon[101970]: pgmap v665: 425 pgs: 425 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 874 KiB/s rd, 831 KiB/s wr, 854 op/s 2024-03-29T05:04:03.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:03 smithi181 ceph-mon[106466]: pgmap v665: 425 pgs: 425 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 874 KiB/s rd, 831 KiB/s wr, 854 op/s 2024-03-29T05:04:05.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:05 smithi187 ceph-mon[93536]: pgmap v666: 425 pgs: 425 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 733 KiB/s rd, 697 KiB/s wr, 717 op/s 2024-03-29T05:04:05.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:05 smithi181 ceph-mon[106466]: pgmap v666: 425 pgs: 425 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 733 KiB/s rd, 697 KiB/s wr, 717 op/s 2024-03-29T05:04:05.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:05 smithi181 ceph-mon[101970]: pgmap v666: 425 pgs: 425 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 733 KiB/s rd, 697 KiB/s wr, 717 op/s 2024-03-29T05:04:07.544 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:07.544 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:07.544 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:07.544 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:07.544 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:07.545 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.545 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:07.545 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.545 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:07.545 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:07.545 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.545 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:07.545 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.545 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:07.545 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:07.545 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.545 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:07.545 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.545 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:07.545 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:07.545 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.546 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:07.546 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.546 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:07.546 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:07.546 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.546 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:07.546 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.546 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:07.546 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:07.546 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:04:07.546 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:04:07.546 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:04:07.546 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:04:07.546 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:04:07.547 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:04:07.547 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:04:07.547 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:04:07.547 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:04:07.547 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:04:07.547 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.547 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:04:07.547 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.547 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:07.547 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:04:07.547 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.547 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:04:07.547 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.548 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:07.548 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:04:07.548 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.548 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:04:07.548 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.548 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:07.548 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:04:07.548 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.548 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:04:07.548 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.548 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:07.548 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:04:07.548 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.548 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:04:07.548 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.549 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:07.549 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:04:07.549 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:07.549 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:07.549 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:07.549 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:07.549 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:07.549 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.549 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:07.549 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:07.549 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:07.549 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.549 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:07.549 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:07.549 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:07.549 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.550 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:07.550 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:07.550 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:07.550 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.550 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:07.550 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:07.550 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:07.550 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.550 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:07.550 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:07.551 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:07.551 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 512~512 = 512 2024-03-29T05:04:07.551 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.551 INFO:tasks.workunit.client.0.smithi181.stdout:aio discard: 1536~512 = 0 2024-03-29T05:04:07.551 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:07.552 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:07.552 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:07.552 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:07.552 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:07.552 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 0 2024-03-29T05:04:07.552 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.552 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.552 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 0 2024-03-29T05:04:07.552 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.552 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.552 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 16384 2024-03-29T05:04:07.552 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.552 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.552 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 16384 2024-03-29T05:04:07.552 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.552 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.553 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:07.553 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:07.553 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 49152 2024-03-29T05:04:07.553 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.553 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.553 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 49152 2024-03-29T05:04:07.553 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.553 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.553 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 65536 2024-03-29T05:04:07.553 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.553 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.554 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 65536 2024-03-29T05:04:07.554 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.554 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.554 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:07.554 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:07.554 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 98304 2024-03-29T05:04:07.554 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.554 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.554 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 98304 2024-03-29T05:04:07.554 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.554 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.554 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 114688 2024-03-29T05:04:07.554 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.554 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.554 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 114688 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 147456 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 147456 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 163840 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 163840 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.555 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:07.556 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:07.556 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 196608 2024-03-29T05:04:07.556 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.556 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.556 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 196608 2024-03-29T05:04:07.556 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.556 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.556 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 212992 2024-03-29T05:04:07.556 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.556 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.556 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 212992 2024-03-29T05:04:07.556 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.556 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.556 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:07.556 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:07.557 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.557 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:07.557 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.557 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:07.557 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:07.557 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.557 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.557 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.557 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:07.557 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.557 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:07.557 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:07.557 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.557 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.557 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.558 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:07.558 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.558 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:07.558 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:07.558 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.558 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.558 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.558 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:07.558 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.558 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:07.558 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:07.558 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.558 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.558 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.558 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:07.558 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:07.559 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.559 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:07.559 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:07.559 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.559 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:07.559 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.559 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:07.559 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:07.559 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.559 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.559 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.559 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:07.559 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:07.559 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:07.559 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:07.560 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:07.560 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:07.560 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:07.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:07 smithi187 ceph-mon[93536]: pgmap v667: 425 pgs: 425 active+clean; 15 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 728 KiB/s rd, 1.0 MiB/s wr, 712 op/s 2024-03-29T05:04:07.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:07 smithi181 ceph-mon[101970]: pgmap v667: 425 pgs: 425 active+clean; 15 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 728 KiB/s rd, 1.0 MiB/s wr, 712 op/s 2024-03-29T05:04:07.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:07 smithi181 ceph-mon[106466]: pgmap v667: 425 pgs: 425 active+clean; 15 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 728 KiB/s rd, 1.0 MiB/s wr, 712 op/s 2024-03-29T05:04:09.205 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:04:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:04:08] "GET /metrics HTTP/1.1" 200 60483 "" "Prometheus/2.43.0" 2024-03-29T05:04:09.639 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:09 smithi181 ceph-mon[106466]: pgmap v668: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.8 MiB/s wr, 927 op/s 2024-03-29T05:04:09.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:09 smithi181 ceph-mon[101970]: pgmap v668: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.8 MiB/s wr, 927 op/s 2024-03-29T05:04:10.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:09 smithi187 ceph-mon[93536]: pgmap v668: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.8 MiB/s wr, 927 op/s 2024-03-29T05:04:11.225 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.225 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:11.225 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:11.225 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:11.225 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:11.225 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:11.225 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.225 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.225 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:11.225 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:11.226 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:11.226 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:11.226 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:11.226 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.226 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.226 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:11.226 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.226 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.226 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:11.226 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.226 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.226 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:11.226 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:11.226 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:11.227 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:11.227 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:11.227 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.227 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.227 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:11.227 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:11.227 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:11.227 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:11.227 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:11.227 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.227 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.227 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:11.227 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:11.227 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:11.227 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:11.228 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:11.228 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.228 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.228 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:11.228 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:11.228 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:11.228 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:11.228 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:11.228 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.228 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.228 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:11.228 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.228 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.228 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:11.228 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.229 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.229 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:11.229 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:11.229 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:11.229 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:11.229 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:11.229 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.229 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.229 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:11.229 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:11.229 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:11.229 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:11.229 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:11.229 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.229 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.229 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:11.230 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:11.230 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:11.230 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:11.230 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:11.230 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.230 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.230 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:11.230 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:11.230 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:11.231 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:11.231 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:11.231 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.231 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.231 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:11.231 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.231 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.231 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:11.231 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.231 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.231 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:11.231 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:11.231 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:11.231 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:11.231 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:11.232 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.232 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.232 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:11.232 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:11.232 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:11.232 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:11.232 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:11.232 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.232 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.232 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:11.232 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:11.232 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:11.232 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:11.232 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:11.232 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.233 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.233 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:11.233 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:11.233 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:11.233 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:11.233 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:11.233 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.233 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.233 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:11.233 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:11.233 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:11.233 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:11.234 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:11.234 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:11.234 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:11.234 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:11.234 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 4 2024-03-29T05:04:11.234 INFO:tasks.workunit.client.0.smithi181.stdout:read: 4 2024-03-29T05:04:11.234 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS2 (11098 ms) 2024-03-29T05:04:11.234 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestIOWithIOHint 2024-03-29T05:04:11.234 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:11.948 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:11 smithi187 ceph-mon[93536]: pgmap v669: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.5 MiB/s wr, 380 op/s 2024-03-29T05:04:11.948 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:04:11.949 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.7", "id": [7, 0]}]: dispatch 2024-03-29T05:04:11.949 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]}]: dispatch 2024-03-29T05:04:11.949 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]}]: dispatch 2024-03-29T05:04:11.949 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1e", "id": [7, 4]}]: dispatch 2024-03-29T05:04:11.949 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1", "id": [7, 0]}]: dispatch 2024-03-29T05:04:11.949 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.c", "id": [7, 2]}]: dispatch 2024-03-29T05:04:11.949 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.14", "id": [7, 5]}]: dispatch 2024-03-29T05:04:11.949 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]: dispatch 2024-03-29T05:04:11.949 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.f", "id": [6, 4]}]: dispatch 2024-03-29T05:04:11.949 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.13", "id": [6, 1]}]: dispatch 2024-03-29T05:04:11.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[101970]: pgmap v669: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.5 MiB/s wr, 380 op/s 2024-03-29T05:04:11.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:04:11.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.7", "id": [7, 0]}]: dispatch 2024-03-29T05:04:11.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]}]: dispatch 2024-03-29T05:04:11.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]}]: dispatch 2024-03-29T05:04:11.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1e", "id": [7, 4]}]: dispatch 2024-03-29T05:04:11.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1", "id": [7, 0]}]: dispatch 2024-03-29T05:04:11.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.c", "id": [7, 2]}]: dispatch 2024-03-29T05:04:11.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.14", "id": [7, 5]}]: dispatch 2024-03-29T05:04:11.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]: dispatch 2024-03-29T05:04:11.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.f", "id": [6, 4]}]: dispatch 2024-03-29T05:04:11.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.13", "id": [6, 1]}]: dispatch 2024-03-29T05:04:11.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[106466]: pgmap v669: 425 pgs: 425 active+clean; 27 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.5 MiB/s wr, 380 op/s 2024-03-29T05:04:11.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:04:11.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.7", "id": [7, 0]}]: dispatch 2024-03-29T05:04:11.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]}]: dispatch 2024-03-29T05:04:11.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]}]: dispatch 2024-03-29T05:04:11.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1e", "id": [7, 4]}]: dispatch 2024-03-29T05:04:11.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1", "id": [7, 0]}]: dispatch 2024-03-29T05:04:11.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.c", "id": [7, 2]}]: dispatch 2024-03-29T05:04:11.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.14", "id": [7, 5]}]: dispatch 2024-03-29T05:04:11.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]: dispatch 2024-03-29T05:04:11.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.f", "id": [6, 4]}]: dispatch 2024-03-29T05:04:11.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.13", "id": [6, 1]}]: dispatch 2024-03-29T05:04:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.7", "id": [7, 0]}]': finished 2024-03-29T05:04:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]}]': finished 2024-03-29T05:04:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]}]': finished 2024-03-29T05:04:12.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1e", "id": [7, 4]}]': finished 2024-03-29T05:04:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1", "id": [7, 0]}]': finished 2024-03-29T05:04:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.c", "id": [7, 2]}]': finished 2024-03-29T05:04:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.14", "id": [7, 5]}]': finished 2024-03-29T05:04:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]': finished 2024-03-29T05:04:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.f", "id": [6, 4]}]': finished 2024-03-29T05:04:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.13", "id": [6, 1]}]': finished 2024-03-29T05:04:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: failed to encode map e647 with expected crc 2024-03-29T05:04:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: failed to encode map e647 with expected crc 2024-03-29T05:04:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: failed to encode map e647 with expected crc 2024-03-29T05:04:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: failed to encode map e647 with expected crc 2024-03-29T05:04:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: osdmap e647: 8 total, 8 up, 8 in 2024-03-29T05:04:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: failed to encode map e647 with expected crc 2024-03-29T05:04:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: failed to encode map e647 with expected crc 2024-03-29T05:04:12.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: failed to encode map e647 with expected crc 2024-03-29T05:04:12.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: failed to encode map e647 with expected crc 2024-03-29T05:04:12.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[106466]: osdmap e648: 8 total, 8 up, 8 in 2024-03-29T05:04:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.7", "id": [7, 0]}]': finished 2024-03-29T05:04:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]}]': finished 2024-03-29T05:04:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]}]': finished 2024-03-29T05:04:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1e", "id": [7, 4]}]': finished 2024-03-29T05:04:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1", "id": [7, 0]}]': finished 2024-03-29T05:04:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.c", "id": [7, 2]}]': finished 2024-03-29T05:04:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.14", "id": [7, 5]}]': finished 2024-03-29T05:04:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]': finished 2024-03-29T05:04:12.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.f", "id": [6, 4]}]': finished 2024-03-29T05:04:12.958 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.13", "id": [6, 1]}]': finished 2024-03-29T05:04:12.958 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: failed to encode map e647 with expected crc 2024-03-29T05:04:12.958 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: failed to encode map e647 with expected crc 2024-03-29T05:04:12.958 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: failed to encode map e647 with expected crc 2024-03-29T05:04:12.958 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: failed to encode map e647 with expected crc 2024-03-29T05:04:12.958 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: osdmap e647: 8 total, 8 up, 8 in 2024-03-29T05:04:12.958 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: failed to encode map e647 with expected crc 2024-03-29T05:04:12.958 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: failed to encode map e647 with expected crc 2024-03-29T05:04:12.958 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: failed to encode map e647 with expected crc 2024-03-29T05:04:12.958 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: failed to encode map e647 with expected crc 2024-03-29T05:04:12.958 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:12 smithi181 ceph-mon[101970]: osdmap e648: 8 total, 8 up, 8 in 2024-03-29T05:04:13.072 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:13.072 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:13.072 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:13.072 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:13.072 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:13.072 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.072 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:13.072 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.072 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:13.072 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:13.072 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.072 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:13.073 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.073 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:13.073 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:13.073 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.073 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:13.073 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.073 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:13.073 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:13.073 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.073 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:13.073 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.073 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:13.073 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:13.073 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.073 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:13.073 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.074 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:13.074 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:13.074 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:04:13.074 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:04:13.074 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:04:13.074 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:04:13.074 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:04:13.074 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:04:13.074 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:04:13.074 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:04:13.074 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:04:13.074 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:04:13.074 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.074 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:04:13.075 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.075 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:13.075 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:04:13.075 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.075 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:04:13.075 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.075 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:13.075 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:04:13.075 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.075 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:04:13.075 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.075 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:13.075 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:04:13.075 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.076 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:04:13.076 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.076 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:13.076 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:04:13.076 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.076 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:04:13.076 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.076 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:13.076 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:04:13.076 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:13.076 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:13.076 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:13.076 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:13.076 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:13.076 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.077 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:13.077 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:13.077 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:13.077 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.077 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:13.077 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:13.077 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:13.077 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.077 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:13.077 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:13.077 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:13.077 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.077 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:13.077 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:13.078 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:13.078 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.078 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:13.078 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:13.078 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:13.078 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 512~512 = 512 2024-03-29T05:04:13.078 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.078 INFO:tasks.workunit.client.0.smithi181.stdout:aio discard: 1536~512 = 0 2024-03-29T05:04:13.078 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:13.078 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:13.078 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:13.078 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:13.078 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:13.078 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 0 2024-03-29T05:04:13.079 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.079 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.079 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 0 2024-03-29T05:04:13.079 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.079 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.079 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 16384 2024-03-29T05:04:13.079 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.079 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.079 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 16384 2024-03-29T05:04:13.079 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.079 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.079 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:13.079 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:13.079 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 49152 2024-03-29T05:04:13.080 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.080 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.080 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 49152 2024-03-29T05:04:13.080 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.080 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.080 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 65536 2024-03-29T05:04:13.080 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.080 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.080 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 65536 2024-03-29T05:04:13.080 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.080 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.080 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:13.080 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:13.081 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 98304 2024-03-29T05:04:13.081 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.081 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.081 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 98304 2024-03-29T05:04:13.081 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.081 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.081 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 114688 2024-03-29T05:04:13.081 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.081 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.081 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 114688 2024-03-29T05:04:13.081 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.081 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.081 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:13.081 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:13.082 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 147456 2024-03-29T05:04:13.082 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.082 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.082 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 147456 2024-03-29T05:04:13.082 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.082 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.082 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 163840 2024-03-29T05:04:13.082 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.082 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.082 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 163840 2024-03-29T05:04:13.082 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.082 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.082 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:13.082 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:13.082 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 196608 2024-03-29T05:04:13.083 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.083 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.083 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 196608 2024-03-29T05:04:13.083 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.083 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.083 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 212992 2024-03-29T05:04:13.083 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.083 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.083 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 212992 2024-03-29T05:04:13.083 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.083 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.083 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:13.083 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:13.084 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.084 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:13.084 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.084 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:13.084 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:13.084 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.084 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.084 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.084 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:13.084 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.084 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:13.084 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:13.085 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.085 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.085 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.085 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:13.085 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.085 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:13.085 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:13.085 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.085 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.085 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.085 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:13.085 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.085 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:13.086 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:13.086 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.086 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.086 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.086 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:13.086 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:13.086 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.086 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:13.086 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:13.086 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.086 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:13.086 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.087 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:13.087 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:13.087 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.087 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.087 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.087 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:13.087 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:13.087 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:13.087 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:13.087 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:13.087 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:13.087 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.7", "id": [7, 0]}]': finished 2024-03-29T05:04:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]}]': finished 2024-03-29T05:04:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]}]': finished 2024-03-29T05:04:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1e", "id": [7, 4]}]': finished 2024-03-29T05:04:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1", "id": [7, 0]}]': finished 2024-03-29T05:04:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.c", "id": [7, 2]}]': finished 2024-03-29T05:04:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.14", "id": [7, 5]}]': finished 2024-03-29T05:04:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.18", "id": [7, 1]}]': finished 2024-03-29T05:04:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.f", "id": [6, 4]}]': finished 2024-03-29T05:04:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.13", "id": [6, 1]}]': finished 2024-03-29T05:04:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: failed to encode map e647 with expected crc 2024-03-29T05:04:13.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: failed to encode map e647 with expected crc 2024-03-29T05:04:13.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: failed to encode map e647 with expected crc 2024-03-29T05:04:13.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: failed to encode map e647 with expected crc 2024-03-29T05:04:13.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: osdmap e647: 8 total, 8 up, 8 in 2024-03-29T05:04:13.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: failed to encode map e647 with expected crc 2024-03-29T05:04:13.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: failed to encode map e647 with expected crc 2024-03-29T05:04:13.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: failed to encode map e647 with expected crc 2024-03-29T05:04:13.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: failed to encode map e647 with expected crc 2024-03-29T05:04:13.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:12 smithi187 ceph-mon[93536]: osdmap e648: 8 total, 8 up, 8 in 2024-03-29T05:04:13.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:13 smithi181 ceph-mon[101970]: pgmap v671: 425 pgs: 1 peering, 424 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.1 MiB/s wr, 1.11k op/s 2024-03-29T05:04:13.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:13 smithi181 ceph-mon[101970]: failed to encode map e648 with expected crc 2024-03-29T05:04:13.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:13 smithi181 ceph-mon[101970]: failed to encode map e648 with expected crc 2024-03-29T05:04:13.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:13 smithi181 ceph-mon[101970]: failed to encode map e648 with expected crc 2024-03-29T05:04:13.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:13 smithi181 ceph-mon[101970]: failed to encode map e648 with expected crc 2024-03-29T05:04:13.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:13 smithi181 ceph-mon[101970]: failed to encode map e648 with expected crc 2024-03-29T05:04:13.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:13 smithi181 ceph-mon[106466]: pgmap v671: 425 pgs: 1 peering, 424 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.1 MiB/s wr, 1.11k op/s 2024-03-29T05:04:13.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:13 smithi181 ceph-mon[106466]: failed to encode map e648 with expected crc 2024-03-29T05:04:13.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:13 smithi181 ceph-mon[106466]: failed to encode map e648 with expected crc 2024-03-29T05:04:13.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:13 smithi181 ceph-mon[106466]: failed to encode map e648 with expected crc 2024-03-29T05:04:13.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:13 smithi181 ceph-mon[106466]: failed to encode map e648 with expected crc 2024-03-29T05:04:13.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:13 smithi181 ceph-mon[106466]: failed to encode map e648 with expected crc 2024-03-29T05:04:14.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:13 smithi187 ceph-mon[93536]: pgmap v671: 425 pgs: 1 peering, 424 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.1 MiB/s wr, 1.11k op/s 2024-03-29T05:04:14.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:13 smithi187 ceph-mon[93536]: failed to encode map e648 with expected crc 2024-03-29T05:04:14.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:13 smithi187 ceph-mon[93536]: failed to encode map e648 with expected crc 2024-03-29T05:04:14.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:13 smithi187 ceph-mon[93536]: failed to encode map e648 with expected crc 2024-03-29T05:04:14.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:13 smithi187 ceph-mon[93536]: failed to encode map e648 with expected crc 2024-03-29T05:04:14.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:13 smithi187 ceph-mon[93536]: failed to encode map e648 with expected crc 2024-03-29T05:04:14.681 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.682 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:14.682 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:14.682 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:14.682 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:14.682 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:14.682 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.682 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.682 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:14.682 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:14.682 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:14.682 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:14.682 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:14.682 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.683 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.683 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:14.683 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.683 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.683 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:14.683 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.683 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.683 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:14.683 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:14.683 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:14.683 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:14.683 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:14.683 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.683 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.684 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:14.684 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:14.684 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:14.684 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:14.684 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:14.684 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.684 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.684 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:14.684 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:14.684 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:14.684 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:14.684 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:14.684 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.684 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.684 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:14.685 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:14.685 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:14.685 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:14.685 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:14.685 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.685 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.685 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:14.685 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.685 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.685 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:14.685 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.685 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.685 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:14.685 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:14.685 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:14.686 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:14.686 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:14.686 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.686 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.686 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:14.686 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:14.686 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:14.686 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:14.686 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:14.686 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.686 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.686 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:14.686 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:14.686 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:14.686 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:14.687 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:14.687 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.687 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.687 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:14.687 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:14.687 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:14.687 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:14.687 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:14.687 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.687 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.687 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:14.687 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.687 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.687 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:14.687 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.688 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.688 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:14.688 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:14.688 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:14.688 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:14.688 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:14.688 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.688 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.688 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:14.688 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:14.688 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:14.688 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:14.688 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:14.688 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.688 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.689 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:14.689 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:14.689 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:14.689 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:14.689 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:14.689 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.689 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.689 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:14.689 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:14.689 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:14.689 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:14.689 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:14.689 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.689 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.690 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:14.690 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:14.690 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:14.690 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:14.690 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:14.690 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:14.690 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:14.690 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestIOWithIOHint (3457 ms) 2024-03-29T05:04:14.690 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestDataPoolIO 2024-03-29T05:04:15.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:15 smithi181 ceph-mon[106466]: pgmap v673: 425 pgs: 1 peering, 424 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.1 MiB/s wr, 1.38k op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-03-29T05:04:15.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:15 smithi181 ceph-mon[101970]: pgmap v673: 425 pgs: 1 peering, 424 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.1 MiB/s wr, 1.38k op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-03-29T05:04:16.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:15 smithi187 ceph-mon[93536]: pgmap v673: 425 pgs: 1 peering, 424 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.1 MiB/s wr, 1.38k op/s; 0 B/s, 0 keys/s, 0 objects/s recovering 2024-03-29T05:04:16.691 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:16.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:16 smithi181 ceph-mon[106466]: failed to encode map e649 with expected crc 2024-03-29T05:04:16.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:16 smithi181 ceph-mon[106466]: failed to encode map e649 with expected crc 2024-03-29T05:04:16.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:16 smithi181 ceph-mon[106466]: failed to encode map e649 with expected crc 2024-03-29T05:04:16.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:16 smithi181 ceph-mon[106466]: osdmap e649: 8 total, 8 up, 8 in 2024-03-29T05:04:16.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:16 smithi181 ceph-mon[106466]: failed to encode map e649 with expected crc 2024-03-29T05:04:16.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:16 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1722240707' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:16.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:16 smithi181 ceph-mon[101970]: failed to encode map e649 with expected crc 2024-03-29T05:04:16.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:16 smithi181 ceph-mon[101970]: failed to encode map e649 with expected crc 2024-03-29T05:04:16.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:16 smithi181 ceph-mon[101970]: failed to encode map e649 with expected crc 2024-03-29T05:04:16.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:16 smithi181 ceph-mon[101970]: osdmap e649: 8 total, 8 up, 8 in 2024-03-29T05:04:16.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:16 smithi181 ceph-mon[101970]: failed to encode map e649 with expected crc 2024-03-29T05:04:16.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:16 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1722240707' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:17.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:16 smithi187 ceph-mon[93536]: failed to encode map e649 with expected crc 2024-03-29T05:04:17.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:16 smithi187 ceph-mon[93536]: failed to encode map e649 with expected crc 2024-03-29T05:04:17.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:16 smithi187 ceph-mon[93536]: failed to encode map e649 with expected crc 2024-03-29T05:04:17.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:16 smithi187 ceph-mon[93536]: osdmap e649: 8 total, 8 up, 8 in 2024-03-29T05:04:17.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:16 smithi187 ceph-mon[93536]: failed to encode map e649 with expected crc 2024-03-29T05:04:17.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:16 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1722240707' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:17.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:17 smithi181 ceph-mon[106466]: pgmap v675: 457 pgs: 8 creating+peering, 24 unknown, 1 peering, 424 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 608 KiB/s wr, 1.42k op/s; 5 B/s, 0 keys/s, 0 objects/s recovering 2024-03-29T05:04:17.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:17 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:04:17.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:17 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1722240707' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:17.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:17 smithi181 ceph-mon[106466]: failed to encode map e650 with expected crc 2024-03-29T05:04:17.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:17 smithi181 ceph-mon[106466]: osdmap e650: 8 total, 8 up, 8 in 2024-03-29T05:04:17.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:17 smithi181 ceph-mon[106466]: failed to encode map e650 with expected crc 2024-03-29T05:04:17.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:17 smithi181 ceph-mon[106466]: failed to encode map e650 with expected crc 2024-03-29T05:04:17.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:17 smithi181 ceph-mon[106466]: failed to encode map e650 with expected crc 2024-03-29T05:04:17.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:17 smithi181 ceph-mon[101970]: pgmap v675: 457 pgs: 8 creating+peering, 24 unknown, 1 peering, 424 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 608 KiB/s wr, 1.42k op/s; 5 B/s, 0 keys/s, 0 objects/s recovering 2024-03-29T05:04:17.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:17 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:04:17.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:17 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1722240707' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:17.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:17 smithi181 ceph-mon[101970]: failed to encode map e650 with expected crc 2024-03-29T05:04:17.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:17 smithi181 ceph-mon[101970]: osdmap e650: 8 total, 8 up, 8 in 2024-03-29T05:04:17.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:17 smithi181 ceph-mon[101970]: failed to encode map e650 with expected crc 2024-03-29T05:04:17.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:17 smithi181 ceph-mon[101970]: failed to encode map e650 with expected crc 2024-03-29T05:04:17.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:17 smithi181 ceph-mon[101970]: failed to encode map e650 with expected crc 2024-03-29T05:04:18.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:17 smithi187 ceph-mon[93536]: pgmap v675: 457 pgs: 8 creating+peering, 24 unknown, 1 peering, 424 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 608 KiB/s wr, 1.42k op/s; 5 B/s, 0 keys/s, 0 objects/s recovering 2024-03-29T05:04:18.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:17 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:04:18.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:17 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1722240707' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:18.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:17 smithi187 ceph-mon[93536]: failed to encode map e650 with expected crc 2024-03-29T05:04:18.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:17 smithi187 ceph-mon[93536]: osdmap e650: 8 total, 8 up, 8 in 2024-03-29T05:04:18.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:17 smithi187 ceph-mon[93536]: failed to encode map e650 with expected crc 2024-03-29T05:04:18.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:17 smithi187 ceph-mon[93536]: failed to encode map e650 with expected crc 2024-03-29T05:04:18.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:17 smithi187 ceph-mon[93536]: failed to encode map e650 with expected crc 2024-03-29T05:04:18.764 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:18.765 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:18.767 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:18.768 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:18.768 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:18.768 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:18.768 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:18.768 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:18.768 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:18.768 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:18.768 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:18.768 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:18.768 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:18.768 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:18.768 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:18.768 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:18.768 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:18.769 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:18.769 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:18.769 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:18.769 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:18.769 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:18.769 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:18.769 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:18.769 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:18.769 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:18.769 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:18.769 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:18.769 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:18.769 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:18.769 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:18.769 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:18.770 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:18.770 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:18.770 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:18.770 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:18.770 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:18.770 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:18.770 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:18.770 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:18.770 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:18.771 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:18.771 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:18.771 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:18.771 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:18.771 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:18.771 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:18.771 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:18.771 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:18.771 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:18.771 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:18.771 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:18.771 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:18.771 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:18.771 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:18.772 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 512~512 = 512 2024-03-29T05:04:18.772 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:18.772 INFO:tasks.workunit.client.0.smithi181.stdout:aio discard: 1536~512 = 0 2024-03-29T05:04:18.772 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:18.772 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:18.772 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:18.772 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:18.772 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:18.772 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:18.772 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:18.772 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestDataPoolIO (4078 ms) 2024-03-29T05:04:18.772 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatch 2024-03-29T05:04:18.772 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:18.787 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatch (27 ms) 2024-03-29T05:04:18.788 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatch 2024-03-29T05:04:18.788 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:18.814 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatch (27 ms) 2024-03-29T05:04:18.814 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccess 2024-03-29T05:04:18.814 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:18.841 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccess (24 ms) 2024-03-29T05:04:18.841 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccess 2024-03-29T05:04:18.841 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:18.863 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccess (25 ms) 2024-03-29T05:04:18.863 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned 2024-03-29T05:04:18.863 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:18.928 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned (65 ms) 2024-03-29T05:04:18.928 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned 2024-03-29T05:04:18.928 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:18.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:18 smithi181 ceph-mon[106466]: failed to encode map e651 with expected crc 2024-03-29T05:04:18.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:18 smithi181 ceph-mon[106466]: osdmap e651: 8 total, 8 up, 8 in 2024-03-29T05:04:18.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:18 smithi181 ceph-mon[106466]: failed to encode map e651 with expected crc 2024-03-29T05:04:18.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:18 smithi181 ceph-mon[101970]: failed to encode map e651 with expected crc 2024-03-29T05:04:18.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:18 smithi181 ceph-mon[101970]: osdmap e651: 8 total, 8 up, 8 in 2024-03-29T05:04:18.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:18 smithi181 ceph-mon[101970]: failed to encode map e651 with expected crc 2024-03-29T05:04:18.996 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned (69 ms) 2024-03-29T05:04:18.997 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLarge 2024-03-29T05:04:18.997 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:19.072 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLarge (75 ms) 2024-03-29T05:04:19.073 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLarge 2024-03-29T05:04:19.073 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:19.126 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLarge (55 ms) 2024-03-29T05:04:19.127 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess 2024-03-29T05:04:19.127 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:19.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:18 smithi187 ceph-mon[93536]: failed to encode map e651 with expected crc 2024-03-29T05:04:19.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:18 smithi187 ceph-mon[93536]: osdmap e651: 8 total, 8 up, 8 in 2024-03-29T05:04:19.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:18 smithi187 ceph-mon[93536]: failed to encode map e651 with expected crc 2024-03-29T05:04:19.205 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:04:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:04:18] "GET /metrics HTTP/1.1" 200 60496 "" "Prometheus/2.43.0" 2024-03-29T05:04:19.388 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess (261 ms) 2024-03-29T05:04:19.388 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess 2024-03-29T05:04:19.388 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:19.728 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess (340 ms) 2024-03-29T05:04:19.728 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers 2024-03-29T05:04:19.728 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:19.752 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers (24 ms) 2024-03-29T05:04:19.752 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVMismatch 2024-03-29T05:04:19.752 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:19.777 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVMismatch (26 ms) 2024-03-29T05:04:19.777 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVSuccess 2024-03-29T05:04:19.777 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:19.800 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVSuccess (22 ms) 2024-03-29T05:04:19.800 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestScatterGatherIO 2024-03-29T05:04:19.800 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:19.825 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestScatterGatherIO (25 ms) 2024-03-29T05:04:19.825 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestEmptyDiscard 2024-03-29T05:04:19.825 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:19.844 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.844 INFO:tasks.workunit.client.0.smithi181.stdout:aio discard: 0~1048576 = 0 2024-03-29T05:04:19.844 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.844 INFO:tasks.workunit.client.0.smithi181.stdout:aio discard: 0~4194304 = 0 2024-03-29T05:04:19.844 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.845 INFO:tasks.workunit.client.0.smithi181.stdout:aio discard: 3145728~1048576 = 0 2024-03-29T05:04:19.845 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestEmptyDiscard (19 ms) 2024-03-29T05:04:19.845 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestFUA 2024-03-29T05:04:19.845 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:19.948 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:19.948 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:19.948 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:19.948 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:19.948 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:19.948 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:19.949 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:19.949 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:19.949 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:19.949 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:19.949 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.949 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:19.949 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.949 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:19.949 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:19.949 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.949 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:19.949 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.949 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:19.949 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:19.949 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.950 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:19.950 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.950 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:19.950 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:19.950 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.950 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:19.950 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.950 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:19.950 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:19.950 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.950 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:19.950 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.950 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:19.950 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:19.950 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.951 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:19.951 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:19.951 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:19.951 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.951 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:19.951 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:19.951 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:19.951 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.951 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:19.951 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:19.951 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:19.951 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.951 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:19.951 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:19.951 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:19.952 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.952 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:19.952 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:19.952 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:19.952 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestFUA (104 ms) 2024-03-29T05:04:19.952 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestIOPP 2024-03-29T05:04:19.952 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:19.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:19 smithi181 ceph-mon[101970]: failed to encode map e651 with expected crc 2024-03-29T05:04:19.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:19 smithi181 ceph-mon[101970]: pgmap v678: 457 pgs: 21 creating+peering, 436 active+clean; 33 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 487 KiB/s rd, 243 KiB/s wr, 1.02k op/s; 5 B/s, 0 keys/s, 0 objects/s recovering 2024-03-29T05:04:19.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:19 smithi181 ceph-mon[101970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:04:19.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:19 smithi181 ceph-mon[101970]: osdmap e652: 8 total, 8 up, 8 in 2024-03-29T05:04:19.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:19 smithi181 ceph-mon[101970]: failed to encode map e652 with expected crc 2024-03-29T05:04:19.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:19 smithi181 ceph-mon[101970]: failed to encode map e652 with expected crc 2024-03-29T05:04:19.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:19 smithi181 ceph-mon[101970]: failed to encode map e652 with expected crc 2024-03-29T05:04:19.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:19 smithi181 ceph-mon[106466]: failed to encode map e651 with expected crc 2024-03-29T05:04:19.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:19 smithi181 ceph-mon[106466]: pgmap v678: 457 pgs: 21 creating+peering, 436 active+clean; 33 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 487 KiB/s rd, 243 KiB/s wr, 1.02k op/s; 5 B/s, 0 keys/s, 0 objects/s recovering 2024-03-29T05:04:19.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:19 smithi181 ceph-mon[106466]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:04:19.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:19 smithi181 ceph-mon[106466]: osdmap e652: 8 total, 8 up, 8 in 2024-03-29T05:04:19.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:19 smithi181 ceph-mon[106466]: failed to encode map e652 with expected crc 2024-03-29T05:04:19.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:19 smithi181 ceph-mon[106466]: failed to encode map e652 with expected crc 2024-03-29T05:04:19.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:19 smithi181 ceph-mon[106466]: failed to encode map e652 with expected crc 2024-03-29T05:04:19.968 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:19.968 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:19.968 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:19.968 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:19.968 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:19.968 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.968 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:19.968 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.968 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:19.968 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:19.969 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.969 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:19.969 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.969 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:19.969 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:19.969 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.969 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:19.969 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.969 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:19.969 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:19.969 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.969 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:19.969 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.969 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:19.969 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:19.970 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.970 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:19.970 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.986 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:19.986 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:19.986 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:04:19.986 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:04:19.986 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:04:19.986 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:04:19.986 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:04:19.986 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:04:19.986 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:04:19.986 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:04:19.987 INFO:tasks.workunit.client.0.smithi181.stdout:start compare and write 2024-03-29T05:04:19.987 INFO:tasks.workunit.client.0.smithi181.stdout:compare and wrote: 512 2024-03-29T05:04:19.987 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.987 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:04:19.987 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.987 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:19.987 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:04:19.987 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.988 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:04:19.988 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.988 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:19.988 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:04:19.988 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.988 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:04:19.988 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.988 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:19.988 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:04:19.988 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.989 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:04:19.989 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.989 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:19.989 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:04:19.990 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.990 INFO:tasks.workunit.client.0.smithi181.stdout:started aio compare and write 2024-03-29T05:04:19.990 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:19.991 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:19.991 INFO:tasks.workunit.client.0.smithi181.stdout:finished aio compare and write 2024-03-29T05:04:19.991 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:19.991 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:19.991 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:19.991 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:19.991 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:19.991 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.991 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:19.991 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:19.991 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:19.992 INFO:tasks.workunit.client.0.smithi181.stdout:finished read 2024-03-29T05:04:19.992 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.992 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:19.992 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:19.992 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:19.992 INFO:tasks.workunit.client.0.smithi181.stdout:finished read 2024-03-29T05:04:19.992 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.992 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:19.992 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:19.993 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:19.993 INFO:tasks.workunit.client.0.smithi181.stdout:finished read 2024-03-29T05:04:19.993 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.993 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:19.993 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:19.993 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:19.993 INFO:tasks.workunit.client.0.smithi181.stdout:finished read 2024-03-29T05:04:19.993 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:19.993 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:19.993 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:19.993 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:19.993 INFO:tasks.workunit.client.0.smithi181.stdout:finished read 2024-03-29T05:04:19.993 INFO:tasks.workunit.client.0.smithi181.stdout:discard: 512~512 2024-03-29T05:04:19.994 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:20.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:19 smithi187 ceph-mon[93536]: failed to encode map e651 with expected crc 2024-03-29T05:04:20.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:19 smithi187 ceph-mon[93536]: pgmap v678: 457 pgs: 21 creating+peering, 436 active+clean; 33 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 487 KiB/s rd, 243 KiB/s wr, 1.02k op/s; 5 B/s, 0 keys/s, 0 objects/s recovering 2024-03-29T05:04:20.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:19 smithi187 ceph-mon[93536]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:04:20.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:19 smithi187 ceph-mon[93536]: osdmap e652: 8 total, 8 up, 8 in 2024-03-29T05:04:20.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:19 smithi187 ceph-mon[93536]: failed to encode map e652 with expected crc 2024-03-29T05:04:20.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:19 smithi187 ceph-mon[93536]: failed to encode map e652 with expected crc 2024-03-29T05:04:20.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:19 smithi187 ceph-mon[93536]: failed to encode map e652 with expected crc 2024-03-29T05:04:20.851 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:20.851 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:20.851 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:20.851 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:20.851 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:20.851 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 0 2024-03-29T05:04:20.851 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.851 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.851 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 0 2024-03-29T05:04:20.852 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.852 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.852 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 16384 2024-03-29T05:04:20.852 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.852 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.852 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 16384 2024-03-29T05:04:20.852 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.852 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.852 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:20.852 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:20.852 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 49152 2024-03-29T05:04:20.852 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.852 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.852 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 49152 2024-03-29T05:04:20.853 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.853 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.853 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 65536 2024-03-29T05:04:20.853 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.853 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.853 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 65536 2024-03-29T05:04:20.853 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.853 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.853 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:20.853 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:20.853 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 98304 2024-03-29T05:04:20.853 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.853 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.853 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 98304 2024-03-29T05:04:20.853 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.854 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.854 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 114688 2024-03-29T05:04:20.854 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.854 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.854 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 114688 2024-03-29T05:04:20.854 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.854 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.854 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:20.854 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:20.854 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 147456 2024-03-29T05:04:20.854 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.855 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.855 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 147456 2024-03-29T05:04:20.855 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.855 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.855 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 163840 2024-03-29T05:04:20.855 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.855 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.855 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 163840 2024-03-29T05:04:20.855 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.855 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.856 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:20.856 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:20.856 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 196608 2024-03-29T05:04:20.856 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.856 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.856 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 196608 2024-03-29T05:04:20.856 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.856 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.857 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 212992 2024-03-29T05:04:20.857 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.857 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.857 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 212992 2024-03-29T05:04:20.857 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.857 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.857 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:20.857 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:20.857 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:20.857 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:20.857 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:20.858 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:20.858 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:20.858 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.858 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.858 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:20.858 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:20.858 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:20.858 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:20.858 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:20.858 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.858 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.858 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:20.859 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:20.859 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:20.859 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:20.859 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:20.859 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.859 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.859 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:20.860 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:20.860 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:20.866 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:20.866 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:20.866 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.866 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.866 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:20.867 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:20.867 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:20.867 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:20.867 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:20.867 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:20.867 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:20.867 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:20.867 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:20.886 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:20.887 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:20.887 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.887 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.887 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:20.887 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:20.887 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:20.906 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:20.906 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:20.906 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.906 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.906 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:20.906 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:20.906 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:20.932 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:20.932 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:20.933 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.933 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.933 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:20.933 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:20.933 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:20.958 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:20.958 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:20.958 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.958 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.958 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:20.959 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:20.959 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:20.959 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:20.959 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:20.959 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:20.959 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:20.959 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:20.959 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:20.996 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:20.996 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:20.996 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:20.996 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:20.996 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:20.997 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:20.997 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:21.033 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:21.033 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:21.033 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:21.033 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:21.033 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:21.033 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:21.033 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:21.080 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:21.081 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:21.081 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:21.081 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:21.081 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:21.081 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:21.081 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:21.125 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:21.125 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:21.125 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:21.125 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:21.125 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:21.126 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:21.126 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:21.126 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:21.126 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:21.126 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:21.126 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:21.126 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:21.126 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:21.189 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:21.189 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:21.189 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:21.189 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:21.189 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:21.189 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:21.189 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:21.246 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:21.246 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:21.246 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:21.246 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:21.247 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:21.247 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:21.247 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:21.314 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:21.315 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:21.315 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:21.315 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:21.315 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:21.315 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:21.315 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:21.381 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:21.381 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:21.381 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:21.381 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:21.381 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:21.381 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:21.381 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:21.381 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:21.382 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:21.382 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:21.382 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:21.382 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:21.382 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:21.467 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:21.467 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:21.467 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:21.467 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:21.467 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:21.467 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:21.467 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:21.542 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:21.543 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:21.543 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:21.543 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:21.543 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:21.543 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:21.543 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:21.630 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:21.630 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:21.630 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:21.630 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:21.630 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:21.630 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:21.631 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:21.717 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:21.718 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:21.718 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:21.718 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:21.718 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:21.718 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:21.719 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:21.719 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:21.719 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:21.719 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:21.719 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestIOPP (1770 ms) 2024-03-29T05:04:21.719 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP 2024-03-29T05:04:21.720 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:21.742 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP (24 ms) 2024-03-29T05:04:21.742 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP 2024-03-29T05:04:21.742 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:21.768 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP (26 ms) 2024-03-29T05:04:21.768 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP 2024-03-29T05:04:21.768 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:21.797 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP (29 ms) 2024-03-29T05:04:21.797 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP 2024-03-29T05:04:21.798 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:21.822 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP (25 ms) 2024-03-29T05:04:21.822 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchPP 2024-03-29T05:04:21.822 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:21.848 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchPP (26 ms) 2024-03-29T05:04:21.848 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchPP 2024-03-29T05:04:21.848 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:21.868 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:21.872 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchPP (24 ms) 2024-03-29T05:04:21.872 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP 2024-03-29T05:04:21.872 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:21.901 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP (29 ms) 2024-03-29T05:04:21.901 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP 2024-03-29T05:04:21.901 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:21.921 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:21.925 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP (24 ms) 2024-03-29T05:04:21.925 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessPP 2024-03-29T05:04:21.925 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:21.949 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessPP (24 ms) 2024-03-29T05:04:21.949 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessPP 2024-03-29T05:04:21.949 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:21.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:21 smithi181 ceph-mon[106466]: pgmap v680: 457 pgs: 21 creating+peering, 436 active+clean; 33 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 590 KiB/s rd, 295 KiB/s wr, 1.24k op/s; 7 B/s, 0 objects/s recovering 2024-03-29T05:04:21.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:21 smithi181 ceph-mon[101970]: pgmap v680: 457 pgs: 21 creating+peering, 436 active+clean; 33 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 590 KiB/s rd, 295 KiB/s wr, 1.24k op/s; 7 B/s, 0 objects/s recovering 2024-03-29T05:04:21.967 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:21.970 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessPP (21 ms) 2024-03-29T05:04:21.970 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP 2024-03-29T05:04:21.970 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:21.996 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP (26 ms) 2024-03-29T05:04:21.996 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP 2024-03-29T05:04:21.996 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:22.016 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:22.020 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP (24 ms) 2024-03-29T05:04:22.020 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP 2024-03-29T05:04:22.020 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:22.098 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP (78 ms) 2024-03-29T05:04:22.098 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP 2024-03-29T05:04:22.098 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:22.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:21 smithi187 ceph-mon[93536]: pgmap v680: 457 pgs: 21 creating+peering, 436 active+clean; 33 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 590 KiB/s rd, 295 KiB/s wr, 1.24k op/s; 7 B/s, 0 objects/s recovering 2024-03-29T05:04:22.147 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:22.164 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP (66 ms) 2024-03-29T05:04:22.164 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLargePP 2024-03-29T05:04:22.164 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:22.239 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLargePP (76 ms) 2024-03-29T05:04:22.239 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLargePP 2024-03-29T05:04:22.240 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:22.288 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:22.307 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLargePP (68 ms) 2024-03-29T05:04:22.307 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP 2024-03-29T05:04:22.307 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:22.609 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP (301 ms) 2024-03-29T05:04:22.609 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP 2024-03-29T05:04:22.609 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:22.901 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:22.922 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP (314 ms) 2024-03-29T05:04:22.922 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestIOPPWithIOHint 2024-03-29T05:04:22.923 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:22.951 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:22.951 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:22.951 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:22.951 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:22.951 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:04:22.951 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:22.951 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:22.952 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:22.952 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:22.952 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:22.952 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:22.952 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:22.952 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:22.953 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:22.953 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:22.953 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:22.953 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:22.953 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:22.954 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:22.954 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:22.954 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:22.954 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:22.954 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:22.957 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:22.957 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:22.957 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:22.957 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:04:22.957 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:22.958 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:22.958 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:04:22.958 INFO:tasks.workunit.client.0.smithi181.stdout:read: 512 2024-03-29T05:04:22.958 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:22.958 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:22.958 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:22.958 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:22.958 INFO:tasks.workunit.client.0.smithi181.stdout:finished read 2024-03-29T05:04:22.958 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:22.959 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:22.959 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:22.959 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:22.959 INFO:tasks.workunit.client.0.smithi181.stdout:finished read 2024-03-29T05:04:22.959 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:22.959 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:22.959 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:22.959 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:22.959 INFO:tasks.workunit.client.0.smithi181.stdout:finished read 2024-03-29T05:04:22.960 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:22.960 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:22.960 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:22.960 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:22.960 INFO:tasks.workunit.client.0.smithi181.stdout:finished read 2024-03-29T05:04:22.960 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:22.960 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:04:22.960 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:04:23.848 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:04:23.848 INFO:tasks.workunit.client.0.smithi181.stdout:finished read 2024-03-29T05:04:23.848 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 0 2024-03-29T05:04:23.849 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.849 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.849 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 0 2024-03-29T05:04:23.849 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.849 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.849 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 16384 2024-03-29T05:04:23.849 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.849 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.849 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 16384 2024-03-29T05:04:23.849 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.849 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.849 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:23.849 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:23.850 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 49152 2024-03-29T05:04:23.850 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.850 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.850 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 49152 2024-03-29T05:04:23.850 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.850 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.850 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 65536 2024-03-29T05:04:23.850 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.850 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.850 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 65536 2024-03-29T05:04:23.850 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.850 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.850 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:23.850 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:23.851 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 98304 2024-03-29T05:04:23.851 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.851 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.851 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 98304 2024-03-29T05:04:23.851 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.851 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.851 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 114688 2024-03-29T05:04:23.851 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.851 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.851 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 114688 2024-03-29T05:04:23.851 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.851 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.851 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:23.851 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:23.852 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 147456 2024-03-29T05:04:23.852 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.852 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.852 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 147456 2024-03-29T05:04:23.852 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.852 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.852 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 163840 2024-03-29T05:04:23.852 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.852 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.852 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 163840 2024-03-29T05:04:23.852 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.852 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.852 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:23.852 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:23.853 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 196608 2024-03-29T05:04:23.853 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.853 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.853 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 196608 2024-03-29T05:04:23.853 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.853 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.853 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 212992 2024-03-29T05:04:23.853 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.853 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.853 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 212992 2024-03-29T05:04:23.853 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.853 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.853 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:23.853 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:23.853 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:23.854 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:23.854 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:23.854 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:23.854 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:23.854 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.854 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.854 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:23.854 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:23.854 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:23.854 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:23.855 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:23.855 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.855 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.855 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:23.855 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:23.855 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:23.858 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:23.858 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:23.858 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.858 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.858 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:23.858 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:23.858 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:23.865 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:23.866 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:23.866 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.866 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.866 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:23.866 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:23.866 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:23.866 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:23.866 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:23.866 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:23.866 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:23.866 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:23.866 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:23.886 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:23.886 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:23.886 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.886 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.886 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:23.886 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:23.886 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:23.907 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:23.907 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:23.907 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.907 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.907 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:23.907 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:23.907 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:23.936 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:23.936 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:23.936 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.936 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.936 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:23.936 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:23.937 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:23.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:23 smithi181 ceph-mon[101970]: pgmap v681: 457 pgs: 457 active+clean; 79 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 8.6 MiB/s wr, 2.62k op/s 2024-03-29T05:04:23.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:23 smithi181 ceph-mon[106466]: pgmap v681: 457 pgs: 457 active+clean; 79 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 8.6 MiB/s wr, 2.62k op/s 2024-03-29T05:04:23.964 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:23.964 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:23.964 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:23.964 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:23.964 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:23.964 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:23.964 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:23.964 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:23.964 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:23.964 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:23.964 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:23.964 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:23.964 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:24.002 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:24.002 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:24.002 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:24.002 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:24.003 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.003 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.003 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:24.042 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:24.042 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:24.042 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:24.042 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:24.042 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.042 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.042 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:24.090 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:24.090 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:24.090 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:24.090 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:24.090 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.090 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.090 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:24.135 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:24.135 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:24.135 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:24.135 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:24.135 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.136 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.136 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:24.136 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.136 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.136 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:24.136 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.136 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.136 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:24.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:23 smithi187 ceph-mon[93536]: pgmap v681: 457 pgs: 457 active+clean; 79 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.3 MiB/s rd, 8.6 MiB/s wr, 2.62k op/s 2024-03-29T05:04:24.196 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:24.196 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:24.196 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:24.196 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:24.196 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.196 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.196 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:24.255 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:24.256 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:24.256 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:24.256 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:24.256 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.256 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.256 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:24.323 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:24.323 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:24.324 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:24.324 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:24.324 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.324 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.324 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:24.385 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:24.385 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:24.385 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:24.385 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:24.385 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.386 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.386 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:24.386 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.386 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.386 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:24.386 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.386 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.386 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:24.467 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:24.467 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:24.467 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:24.467 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:24.467 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.467 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.467 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:24.541 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:24.542 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:24.542 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:24.542 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:24.542 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.542 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.542 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:24.628 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:24.628 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:24.628 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:24.628 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:24.629 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.629 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.629 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:04:24.714 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:04:24.715 INFO:tasks.workunit.client.0.smithi181.stdout:finished writesame 2024-03-29T05:04:24.715 INFO:tasks.workunit.client.0.smithi181.stdout:to verify the data 2024-03-29T05:04:24.715 INFO:tasks.workunit.client.0.smithi181.stdout:verified 2024-03-29T05:04:24.715 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.715 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.715 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:24.716 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:04:24.716 INFO:tasks.workunit.client.0.smithi181.stdout:started writesame 2024-03-29T05:04:24.716 INFO:tasks.workunit.client.0.smithi181.stdout:expected fail, finished writesame 2024-03-29T05:04:24.716 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestIOPPWithIOHint (1791 ms) 2024-03-29T05:04:24.716 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestIOToSnapshot 2024-03-29T05:04:24.716 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:26.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:25 smithi187 ceph-mon[93536]: pgmap v682: 457 pgs: 457 active+clean; 95 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 9.1 MiB/s wr, 2.03k op/s 2024-03-29T05:04:26.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:25 smithi181 ceph-mon[101970]: pgmap v682: 457 pgs: 457 active+clean; 95 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 9.1 MiB/s wr, 2.03k op/s 2024-03-29T05:04:26.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:25 smithi181 ceph-mon[106466]: pgmap v682: 457 pgs: 457 active+clean; 95 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 9.1 MiB/s wr, 2.03k op/s 2024-03-29T05:04:26.756 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 0 2024-03-29T05:04:26.756 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 0 2024-03-29T05:04:26.756 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 1 2024-03-29T05:04:26.756 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 1 2024-03-29T05:04:26.756 INFO:tasks.workunit.client.0.smithi181.stdout:snap: orig 2024-03-29T05:04:26.756 INFO:tasks.workunit.client.0.smithi181.stdout:found orig with size 2097152 2024-03-29T05:04:26.756 INFO:tasks.workunit.client.0.smithi181.stdout:read: 80 2024-03-29T05:04:26.756 INFO:tasks.workunit.client.0.smithi181.stdout:write test data! 2024-03-29T05:04:26.756 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 80 2024-03-29T05:04:26.756 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 2 2024-03-29T05:04:26.756 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 2 2024-03-29T05:04:26.756 INFO:tasks.workunit.client.0.smithi181.stdout:snap: orig 2024-03-29T05:04:26.756 INFO:tasks.workunit.client.0.smithi181.stdout:snap: written 2024-03-29T05:04:26.756 INFO:tasks.workunit.client.0.smithi181.stdout:found orig with size 2097152 2024-03-29T05:04:26.757 INFO:tasks.workunit.client.0.smithi181.stdout:found written with size 2097152 2024-03-29T05:04:26.757 INFO:tasks.workunit.client.0.smithi181.stdout:read: 80 2024-03-29T05:04:26.757 INFO:tasks.workunit.client.0.smithi181.stdout:read: 80 2024-03-29T05:04:26.757 INFO:tasks.workunit.client.0.smithi181.stdout:read: 80 2024-03-29T05:04:26.757 INFO:tasks.workunit.client.0.smithi181.stdout:write to snapshot returned -30 2024-03-29T05:04:26.757 INFO:tasks.workunit.client.0.smithi181.stdout:Read-only file system 2024-03-29T05:04:26.777 INFO:tasks.workunit.client.0.smithi181.stdout:read: 80 2024-03-29T05:04:26.777 INFO:tasks.workunit.client.0.smithi181.stdout:read: 80 2024-03-29T05:04:26.777 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 80 2024-03-29T05:04:26.777 INFO:tasks.workunit.client.0.smithi181.stdout:opening testimg@orig 2024-03-29T05:04:26.777 INFO:tasks.workunit.client.0.smithi181.stdout:read: 80 2024-03-29T05:04:26.777 INFO:tasks.workunit.client.0.smithi181.stdout:write to snapshot returned -30 2024-03-29T05:04:26.778 INFO:tasks.workunit.client.0.smithi181.stdout:Read-only file system 2024-03-29T05:04:27.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:26 smithi187 ceph-mon[93536]: failed to encode map e653 with expected crc 2024-03-29T05:04:27.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:26 smithi187 ceph-mon[93536]: osdmap e653: 8 total, 8 up, 8 in 2024-03-29T05:04:27.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:04:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:26 smithi181 ceph-mon[101970]: failed to encode map e653 with expected crc 2024-03-29T05:04:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:26 smithi181 ceph-mon[101970]: osdmap e653: 8 total, 8 up, 8 in 2024-03-29T05:04:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:04:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:26 smithi181 ceph-mon[106466]: failed to encode map e653 with expected crc 2024-03-29T05:04:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:26 smithi181 ceph-mon[106466]: osdmap e653: 8 total, 8 up, 8 in 2024-03-29T05:04:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:04:28.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:27 smithi187 ceph-mon[93536]: pgmap v684: 457 pgs: 457 active+clean; 111 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 11 MiB/s wr, 1.28k op/s 2024-03-29T05:04:28.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:27 smithi187 ceph-mon[93536]: osdmap e654: 8 total, 8 up, 8 in 2024-03-29T05:04:28.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:27 smithi187 ceph-mon[93536]: failed to encode map e654 with expected crc 2024-03-29T05:04:28.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:27 smithi187 ceph-mon[93536]: failed to encode map e654 with expected crc 2024-03-29T05:04:28.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:27 smithi187 ceph-mon[93536]: osdmap e655: 8 total, 8 up, 8 in 2024-03-29T05:04:28.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:27 smithi181 ceph-mon[101970]: pgmap v684: 457 pgs: 457 active+clean; 111 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 11 MiB/s wr, 1.28k op/s 2024-03-29T05:04:28.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:27 smithi181 ceph-mon[101970]: osdmap e654: 8 total, 8 up, 8 in 2024-03-29T05:04:28.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:27 smithi181 ceph-mon[101970]: failed to encode map e654 with expected crc 2024-03-29T05:04:28.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:27 smithi181 ceph-mon[101970]: failed to encode map e654 with expected crc 2024-03-29T05:04:28.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:27 smithi181 ceph-mon[101970]: osdmap e655: 8 total, 8 up, 8 in 2024-03-29T05:04:28.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:27 smithi181 ceph-mon[106466]: pgmap v684: 457 pgs: 457 active+clean; 111 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 11 MiB/s wr, 1.28k op/s 2024-03-29T05:04:28.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:27 smithi181 ceph-mon[106466]: osdmap e654: 8 total, 8 up, 8 in 2024-03-29T05:04:28.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:27 smithi181 ceph-mon[106466]: failed to encode map e654 with expected crc 2024-03-29T05:04:28.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:27 smithi181 ceph-mon[106466]: failed to encode map e654 with expected crc 2024-03-29T05:04:28.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:27 smithi181 ceph-mon[106466]: osdmap e655: 8 total, 8 up, 8 in 2024-03-29T05:04:28.660 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 2 2024-03-29T05:04:28.660 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 2 2024-03-29T05:04:28.660 INFO:tasks.workunit.client.0.smithi181.stdout:snap: orig 2024-03-29T05:04:28.660 INFO:tasks.workunit.client.0.smithi181.stdout:snap: written 2024-03-29T05:04:28.660 INFO:tasks.workunit.client.0.smithi181.stdout:found orig with size 2097152 2024-03-29T05:04:28.660 INFO:tasks.workunit.client.0.smithi181.stdout:found written with size 2097152 2024-03-29T05:04:28.661 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 1 2024-03-29T05:04:28.661 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 1 2024-03-29T05:04:28.661 INFO:tasks.workunit.client.0.smithi181.stdout:snap: orig 2024-03-29T05:04:28.661 INFO:tasks.workunit.client.0.smithi181.stdout:found orig with size 2097152 2024-03-29T05:04:28.661 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 0 2024-03-29T05:04:28.661 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 0 2024-03-29T05:04:28.661 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestIOToSnapshot (3946 ms) 2024-03-29T05:04:28.661 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestSnapshotDeletedIo 2024-03-29T05:04:28.661 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:29.037 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:28 smithi187 ceph-mon[93536]: failed to encode map e655 with expected crc 2024-03-29T05:04:29.037 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:28 smithi187 ceph-mon[93536]: failed to encode map e655 with expected crc 2024-03-29T05:04:29.037 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:28 smithi187 ceph-mon[93536]: failed to encode map e655 with expected crc 2024-03-29T05:04:29.037 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:28 smithi187 ceph-mon[93536]: osdmap e656: 8 total, 8 up, 8 in 2024-03-29T05:04:29.205 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:04:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:04:28] "GET /metrics HTTP/1.1" 200 62907 "" "Prometheus/2.43.0" 2024-03-29T05:04:29.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:28 smithi181 ceph-mon[106466]: failed to encode map e655 with expected crc 2024-03-29T05:04:29.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:28 smithi181 ceph-mon[106466]: failed to encode map e655 with expected crc 2024-03-29T05:04:29.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:28 smithi181 ceph-mon[106466]: failed to encode map e655 with expected crc 2024-03-29T05:04:29.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:28 smithi181 ceph-mon[106466]: osdmap e656: 8 total, 8 up, 8 in 2024-03-29T05:04:29.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:28 smithi181 ceph-mon[101970]: failed to encode map e655 with expected crc 2024-03-29T05:04:29.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:28 smithi181 ceph-mon[101970]: failed to encode map e655 with expected crc 2024-03-29T05:04:29.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:28 smithi181 ceph-mon[101970]: failed to encode map e655 with expected crc 2024-03-29T05:04:29.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:28 smithi181 ceph-mon[101970]: osdmap e656: 8 total, 8 up, 8 in 2024-03-29T05:04:30.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:29 smithi187 ceph-mon[93536]: pgmap v687: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 11 MiB/s wr, 1.65k op/s 2024-03-29T05:04:30.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:29 smithi187 ceph-mon[93536]: failed to encode map e656 with expected crc 2024-03-29T05:04:30.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:29 smithi187 ceph-mon[93536]: failed to encode map e656 with expected crc 2024-03-29T05:04:30.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:29 smithi187 ceph-mon[93536]: osdmap e657: 8 total, 8 up, 8 in 2024-03-29T05:04:30.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:29 smithi181 ceph-mon[101970]: pgmap v687: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 11 MiB/s wr, 1.65k op/s 2024-03-29T05:04:30.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:29 smithi181 ceph-mon[101970]: failed to encode map e656 with expected crc 2024-03-29T05:04:30.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:29 smithi181 ceph-mon[101970]: failed to encode map e656 with expected crc 2024-03-29T05:04:30.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:29 smithi181 ceph-mon[101970]: osdmap e657: 8 total, 8 up, 8 in 2024-03-29T05:04:30.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:29 smithi181 ceph-mon[106466]: pgmap v687: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 11 MiB/s wr, 1.65k op/s 2024-03-29T05:04:30.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:29 smithi181 ceph-mon[106466]: failed to encode map e656 with expected crc 2024-03-29T05:04:30.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:29 smithi181 ceph-mon[106466]: failed to encode map e656 with expected crc 2024-03-29T05:04:30.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:29 smithi181 ceph-mon[106466]: osdmap e657: 8 total, 8 up, 8 in 2024-03-29T05:04:30.666 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestSnapshotDeletedIo (2006 ms) 2024-03-29T05:04:30.666 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestClone 2024-03-29T05:04:30.666 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:31.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:30 smithi187 ceph-mon[93536]: failed to encode map e657 with expected crc 2024-03-29T05:04:31.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:30 smithi187 ceph-mon[93536]: failed to encode map e657 with expected crc 2024-03-29T05:04:31.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:30 smithi187 ceph-mon[93536]: osdmap e658: 8 total, 8 up, 8 in 2024-03-29T05:04:31.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:30 smithi181 ceph-mon[101970]: failed to encode map e657 with expected crc 2024-03-29T05:04:31.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:30 smithi181 ceph-mon[101970]: failed to encode map e657 with expected crc 2024-03-29T05:04:31.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:30 smithi181 ceph-mon[101970]: osdmap e658: 8 total, 8 up, 8 in 2024-03-29T05:04:31.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:30 smithi181 ceph-mon[106466]: failed to encode map e657 with expected crc 2024-03-29T05:04:31.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:30 smithi181 ceph-mon[106466]: failed to encode map e657 with expected crc 2024-03-29T05:04:31.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:30 smithi181 ceph-mon[106466]: osdmap e658: 8 total, 8 up, 8 in 2024-03-29T05:04:32.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:31 smithi187 ceph-mon[93536]: pgmap v690: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.1 MiB/s rd, 6.5 MiB/s wr, 2.34k op/s 2024-03-29T05:04:32.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:31 smithi187 ceph-mon[93536]: failed to encode map e658 with expected crc 2024-03-29T05:04:32.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:31 smithi187 ceph-mon[93536]: failed to encode map e658 with expected crc 2024-03-29T05:04:32.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:31 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:32.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:31 smithi181 ceph-mon[101970]: pgmap v690: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.1 MiB/s rd, 6.5 MiB/s wr, 2.34k op/s 2024-03-29T05:04:32.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:31 smithi181 ceph-mon[101970]: failed to encode map e658 with expected crc 2024-03-29T05:04:32.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:31 smithi181 ceph-mon[101970]: failed to encode map e658 with expected crc 2024-03-29T05:04:32.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:31 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:32.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:31 smithi181 ceph-mon[106466]: pgmap v690: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.1 MiB/s rd, 6.5 MiB/s wr, 2.34k op/s 2024-03-29T05:04:32.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:31 smithi181 ceph-mon[106466]: failed to encode map e658 with expected crc 2024-03-29T05:04:32.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:31 smithi181 ceph-mon[106466]: failed to encode map e658 with expected crc 2024-03-29T05:04:32.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:31 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:32.774 INFO:tasks.workunit.client.0.smithi181.stdout:made parent image "parent" 2024-03-29T05:04:32.774 INFO:tasks.workunit.client.0.smithi181.stdout:parent has no parent info 2024-03-29T05:04:32.774 INFO:tasks.workunit.client.0.smithi181.stdout:made snapshot "parent@parent_snap" 2024-03-29T05:04:32.774 INFO:tasks.workunit.client.0.smithi181.stdout:can't unprotect an unprotected snap 2024-03-29T05:04:32.774 INFO:tasks.workunit.client.0.smithi181.stdout:can't protect a protected snap 2024-03-29T05:04:32.775 INFO:tasks.workunit.client.0.smithi181.stdout:made and opened clone "child" 2024-03-29T05:04:32.775 INFO:tasks.workunit.client.0.smithi181.stdout:read: 8 2024-03-29T05:04:32.775 INFO:tasks.workunit.client.0.smithi181.stdout:read: 8 2024-03-29T05:04:32.775 INFO:tasks.workunit.client.0.smithi181.stdout:read: 8 2024-03-29T05:04:32.775 INFO:tasks.workunit.client.0.smithi181.stdout:sizes and overlaps are good between parent and child 2024-03-29T05:04:32.775 INFO:tasks.workunit.client.0.smithi181.stdout:child image successfully cloned all image-meta pairs 2024-03-29T05:04:32.775 INFO:tasks.workunit.client.0.smithi181.stdout:sized down clone, changed overlap 2024-03-29T05:04:32.775 INFO:tasks.workunit.client.0.smithi181.stdout:parent info: size 4194304 obj_size 4194304 parent_pool 18446744073709551615 2024-03-29T05:04:32.775 INFO:tasks.workunit.client.0.smithi181.stdout:sized up clone, changed size but not overlap or parent's size 2024-03-29T05:04:32.775 INFO:tasks.workunit.client.0.smithi181.stdout:can't remove parent while child still exists 2024-03-29T05:04:32.775 INFO:tasks.workunit.client.0.smithi181.stdout:can't remove parent while still protected 2024-03-29T05:04:32.776 INFO:tasks.workunit.client.0.smithi181.stdout:removed parent snap after unprotecting 2024-03-29T05:04:32.776 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestClone (2107 ms) 2024-03-29T05:04:32.776 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestClone2 2024-03-29T05:04:32.776 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:33.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:32 smithi187 ceph-mon[93536]: osdmap e659: 8 total, 8 up, 8 in 2024-03-29T05:04:33.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:32 smithi187 ceph-mon[93536]: failed to encode map e659 with expected crc 2024-03-29T05:04:33.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:32 smithi181 ceph-mon[101970]: osdmap e659: 8 total, 8 up, 8 in 2024-03-29T05:04:33.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:32 smithi181 ceph-mon[101970]: failed to encode map e659 with expected crc 2024-03-29T05:04:33.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:32 smithi181 ceph-mon[106466]: osdmap e659: 8 total, 8 up, 8 in 2024-03-29T05:04:33.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:32 smithi181 ceph-mon[106466]: failed to encode map e659 with expected crc 2024-03-29T05:04:33.877 INFO:tasks.workunit.client.0.smithi181.stdout:made parent image "parent" 2024-03-29T05:04:33.878 INFO:tasks.workunit.client.0.smithi181.stdout:parent has no parent info 2024-03-29T05:04:33.878 INFO:tasks.workunit.client.0.smithi181.stdout:made snapshot "parent@parent_snap" 2024-03-29T05:04:33.878 INFO:tasks.workunit.client.0.smithi181.stdout:made and opened clone "child" 2024-03-29T05:04:33.878 INFO:tasks.workunit.client.0.smithi181.stdout:child image successfully cloned all image-meta pairs 2024-03-29T05:04:33.878 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestClone2 (1105 ms) 2024-03-29T05:04:33.878 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ListChildren 2024-03-29T05:04:34.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:33 smithi187 ceph-mon[93536]: pgmap v693: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 442 KiB/s rd, 31 KiB/s wr, 532 op/s 2024-03-29T05:04:34.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:33 smithi187 ceph-mon[93536]: failed to encode map e660 with expected crc 2024-03-29T05:04:34.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:33 smithi187 ceph-mon[93536]: osdmap e660: 8 total, 8 up, 8 in 2024-03-29T05:04:34.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:33 smithi187 ceph-mon[93536]: failed to encode map e660 with expected crc 2024-03-29T05:04:34.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:33 smithi187 ceph-mon[93536]: failed to encode map e660 with expected crc 2024-03-29T05:04:34.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:33 smithi181 ceph-mon[101970]: pgmap v693: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 442 KiB/s rd, 31 KiB/s wr, 532 op/s 2024-03-29T05:04:34.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:33 smithi181 ceph-mon[101970]: failed to encode map e660 with expected crc 2024-03-29T05:04:34.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:33 smithi181 ceph-mon[101970]: osdmap e660: 8 total, 8 up, 8 in 2024-03-29T05:04:34.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:33 smithi181 ceph-mon[101970]: failed to encode map e660 with expected crc 2024-03-29T05:04:34.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:33 smithi181 ceph-mon[101970]: failed to encode map e660 with expected crc 2024-03-29T05:04:34.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:33 smithi181 ceph-mon[106466]: pgmap v693: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 442 KiB/s rd, 31 KiB/s wr, 532 op/s 2024-03-29T05:04:34.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:33 smithi181 ceph-mon[106466]: failed to encode map e660 with expected crc 2024-03-29T05:04:34.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:33 smithi181 ceph-mon[106466]: osdmap e660: 8 total, 8 up, 8 in 2024-03-29T05:04:34.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:33 smithi181 ceph-mon[106466]: failed to encode map e660 with expected crc 2024-03-29T05:04:34.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:33 smithi181 ceph-mon[106466]: failed to encode map e660 with expected crc 2024-03-29T05:04:35.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:34 smithi187 ceph-mon[93536]: failed to encode map e661 with expected crc 2024-03-29T05:04:35.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:34 smithi187 ceph-mon[93536]: osdmap e661: 8 total, 8 up, 8 in 2024-03-29T05:04:35.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:34 smithi187 ceph-mon[93536]: failed to encode map e661 with expected crc 2024-03-29T05:04:35.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:34 smithi187 ceph-mon[93536]: failed to encode map e661 with expected crc 2024-03-29T05:04:35.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:34 smithi181 ceph-mon[101970]: failed to encode map e661 with expected crc 2024-03-29T05:04:35.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:34 smithi181 ceph-mon[101970]: osdmap e661: 8 total, 8 up, 8 in 2024-03-29T05:04:35.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:34 smithi181 ceph-mon[101970]: failed to encode map e661 with expected crc 2024-03-29T05:04:35.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:34 smithi181 ceph-mon[101970]: failed to encode map e661 with expected crc 2024-03-29T05:04:35.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:34 smithi181 ceph-mon[106466]: failed to encode map e661 with expected crc 2024-03-29T05:04:35.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:34 smithi181 ceph-mon[106466]: osdmap e661: 8 total, 8 up, 8 in 2024-03-29T05:04:35.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:34 smithi181 ceph-mon[106466]: failed to encode map e661 with expected crc 2024-03-29T05:04:35.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:34 smithi181 ceph-mon[106466]: failed to encode map e661 with expected crc 2024-03-29T05:04:36.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:35 smithi187 ceph-mon[93536]: pgmap v696: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 704 KiB/s rd, 49 KiB/s wr, 842 op/s 2024-03-29T05:04:36.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:35 smithi187 ceph-mon[93536]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:35 smithi187 ceph-mon[93536]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:35 smithi187 ceph-mon[93536]: osdmap e662: 8 total, 8 up, 8 in 2024-03-29T05:04:36.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:35 smithi187 ceph-mon[93536]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:35 smithi187 ceph-mon[93536]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:35 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/982009931' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:36.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:35 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:36.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:35 smithi187 ceph-mon[93536]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:35 smithi187 ceph-mon[93536]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[101970]: pgmap v696: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 704 KiB/s rd, 49 KiB/s wr, 842 op/s 2024-03-29T05:04:36.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[101970]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[101970]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[101970]: osdmap e662: 8 total, 8 up, 8 in 2024-03-29T05:04:36.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[101970]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[101970]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/982009931' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:36.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:36.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[101970]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[101970]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[106466]: pgmap v696: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 704 KiB/s rd, 49 KiB/s wr, 842 op/s 2024-03-29T05:04:36.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[106466]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[106466]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[106466]: osdmap e662: 8 total, 8 up, 8 in 2024-03-29T05:04:36.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[106466]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[106466]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/982009931' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:36.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:36.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[106466]: failed to encode map e662 with expected crc 2024-03-29T05:04:36.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:35 smithi181 ceph-mon[106466]: failed to encode map e662 with expected crc 2024-03-29T05:04:37.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:36 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:37.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:36 smithi187 ceph-mon[93536]: failed to encode map e663 with expected crc 2024-03-29T05:04:37.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:36 smithi187 ceph-mon[93536]: failed to encode map e663 with expected crc 2024-03-29T05:04:37.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:36 smithi187 ceph-mon[93536]: osdmap e663: 8 total, 8 up, 8 in 2024-03-29T05:04:37.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:36 smithi187 ceph-mon[93536]: failed to encode map e663 with expected crc 2024-03-29T05:04:37.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:36 smithi187 ceph-mon[93536]: failed to encode map e663 with expected crc 2024-03-29T05:04:37.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:36 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:04:37.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:36 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:37.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:36 smithi181 ceph-mon[106466]: failed to encode map e663 with expected crc 2024-03-29T05:04:37.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:36 smithi181 ceph-mon[106466]: failed to encode map e663 with expected crc 2024-03-29T05:04:37.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:36 smithi181 ceph-mon[106466]: osdmap e663: 8 total, 8 up, 8 in 2024-03-29T05:04:37.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:36 smithi181 ceph-mon[106466]: failed to encode map e663 with expected crc 2024-03-29T05:04:37.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:36 smithi181 ceph-mon[106466]: failed to encode map e663 with expected crc 2024-03-29T05:04:37.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:36 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:04:37.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:36 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:37.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:36 smithi181 ceph-mon[101970]: failed to encode map e663 with expected crc 2024-03-29T05:04:37.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:36 smithi181 ceph-mon[101970]: failed to encode map e663 with expected crc 2024-03-29T05:04:37.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:36 smithi181 ceph-mon[101970]: osdmap e663: 8 total, 8 up, 8 in 2024-03-29T05:04:37.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:36 smithi181 ceph-mon[101970]: failed to encode map e663 with expected crc 2024-03-29T05:04:37.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:36 smithi181 ceph-mon[101970]: failed to encode map e663 with expected crc 2024-03-29T05:04:37.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:36 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:04:37.650 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:38.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:37 smithi187 ceph-mon[93536]: pgmap v699: 489 pgs: 21 unknown, 468 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 272 KiB/s rd, 21 KiB/s wr, 329 op/s 2024-03-29T05:04:38.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:37 smithi187 ceph-mon[93536]: failed to encode map e664 with expected crc 2024-03-29T05:04:38.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:37 smithi187 ceph-mon[93536]: failed to encode map e664 with expected crc 2024-03-29T05:04:38.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:37 smithi187 ceph-mon[93536]: failed to encode map e664 with expected crc 2024-03-29T05:04:38.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:37 smithi187 ceph-mon[93536]: osdmap e664: 8 total, 8 up, 8 in 2024-03-29T05:04:38.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:37 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2456955443' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:38.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:37 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:38.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:37 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:38.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:37 smithi187 ceph-mon[93536]: osdmap e665: 8 total, 8 up, 8 in 2024-03-29T05:04:38.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[101970]: pgmap v699: 489 pgs: 21 unknown, 468 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 272 KiB/s rd, 21 KiB/s wr, 329 op/s 2024-03-29T05:04:38.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[101970]: failed to encode map e664 with expected crc 2024-03-29T05:04:38.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[101970]: failed to encode map e664 with expected crc 2024-03-29T05:04:38.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[101970]: failed to encode map e664 with expected crc 2024-03-29T05:04:38.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[101970]: osdmap e664: 8 total, 8 up, 8 in 2024-03-29T05:04:38.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2456955443' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:38.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:38.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:38.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[101970]: osdmap e665: 8 total, 8 up, 8 in 2024-03-29T05:04:38.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[106466]: pgmap v699: 489 pgs: 21 unknown, 468 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 272 KiB/s rd, 21 KiB/s wr, 329 op/s 2024-03-29T05:04:38.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[106466]: failed to encode map e664 with expected crc 2024-03-29T05:04:38.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[106466]: failed to encode map e664 with expected crc 2024-03-29T05:04:38.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[106466]: failed to encode map e664 with expected crc 2024-03-29T05:04:38.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[106466]: osdmap e664: 8 total, 8 up, 8 in 2024-03-29T05:04:38.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2456955443' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:38.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:38.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:38.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:37 smithi181 ceph-mon[106466]: osdmap e665: 8 total, 8 up, 8 in 2024-03-29T05:04:38.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:38 smithi181 ceph-mon[101970]: failed to encode map e665 with expected crc 2024-03-29T05:04:38.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:38 smithi181 ceph-mon[101970]: failed to encode map e665 with expected crc 2024-03-29T05:04:38.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:38 smithi181 ceph-mon[101970]: failed to encode map e665 with expected crc 2024-03-29T05:04:38.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:38 smithi181 ceph-mon[101970]: failed to encode map e665 with expected crc 2024-03-29T05:04:38.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:38 smithi181 ceph-mon[101970]: osdmap e666: 8 total, 8 up, 8 in 2024-03-29T05:04:38.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:38 smithi181 ceph-mon[106466]: failed to encode map e665 with expected crc 2024-03-29T05:04:38.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:38 smithi181 ceph-mon[106466]: failed to encode map e665 with expected crc 2024-03-29T05:04:38.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:38 smithi181 ceph-mon[106466]: failed to encode map e665 with expected crc 2024-03-29T05:04:38.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:38 smithi181 ceph-mon[106466]: failed to encode map e665 with expected crc 2024-03-29T05:04:38.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:38 smithi181 ceph-mon[106466]: osdmap e666: 8 total, 8 up, 8 in 2024-03-29T05:04:39.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:38 smithi187 ceph-mon[93536]: failed to encode map e665 with expected crc 2024-03-29T05:04:39.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:38 smithi187 ceph-mon[93536]: failed to encode map e665 with expected crc 2024-03-29T05:04:39.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:38 smithi187 ceph-mon[93536]: failed to encode map e665 with expected crc 2024-03-29T05:04:39.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:38 smithi187 ceph-mon[93536]: failed to encode map e665 with expected crc 2024-03-29T05:04:39.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:38 smithi187 ceph-mon[93536]: osdmap e666: 8 total, 8 up, 8 in 2024-03-29T05:04:39.205 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:04:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:04:38] "GET /metrics HTTP/1.1" 200 62907 "" "Prometheus/2.43.0" 2024-03-29T05:04:39.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:39 smithi181 ceph-mon[106466]: pgmap v702: 521 pgs: 9 creating+peering, 19 unknown, 493 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 107 KiB/s rd, 10 KiB/s wr, 137 op/s 2024-03-29T05:04:39.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:39 smithi181 ceph-mon[106466]: failed to encode map e666 with expected crc 2024-03-29T05:04:39.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:39 smithi181 ceph-mon[106466]: failed to encode map e666 with expected crc 2024-03-29T05:04:39.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:39 smithi181 ceph-mon[106466]: failed to encode map e666 with expected crc 2024-03-29T05:04:39.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:39 smithi181 ceph-mon[106466]: osdmap e667: 8 total, 8 up, 8 in 2024-03-29T05:04:39.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:39 smithi181 ceph-mon[101970]: pgmap v702: 521 pgs: 9 creating+peering, 19 unknown, 493 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 107 KiB/s rd, 10 KiB/s wr, 137 op/s 2024-03-29T05:04:39.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:39 smithi181 ceph-mon[101970]: failed to encode map e666 with expected crc 2024-03-29T05:04:39.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:39 smithi181 ceph-mon[101970]: failed to encode map e666 with expected crc 2024-03-29T05:04:39.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:39 smithi181 ceph-mon[101970]: failed to encode map e666 with expected crc 2024-03-29T05:04:39.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:39 smithi181 ceph-mon[101970]: osdmap e667: 8 total, 8 up, 8 in 2024-03-29T05:04:40.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:39 smithi187 ceph-mon[93536]: pgmap v702: 521 pgs: 9 creating+peering, 19 unknown, 493 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 107 KiB/s rd, 10 KiB/s wr, 137 op/s 2024-03-29T05:04:40.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:39 smithi187 ceph-mon[93536]: failed to encode map e666 with expected crc 2024-03-29T05:04:40.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:39 smithi187 ceph-mon[93536]: failed to encode map e666 with expected crc 2024-03-29T05:04:40.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:39 smithi187 ceph-mon[93536]: failed to encode map e666 with expected crc 2024-03-29T05:04:40.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:39 smithi187 ceph-mon[93536]: osdmap e667: 8 total, 8 up, 8 in 2024-03-29T05:04:40.840 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:40 smithi181 ceph-mon[106466]: failed to encode map e667 with expected crc 2024-03-29T05:04:40.841 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:40 smithi181 ceph-mon[106466]: failed to encode map e667 with expected crc 2024-03-29T05:04:40.841 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:40 smithi181 ceph-mon[106466]: failed to encode map e667 with expected crc 2024-03-29T05:04:40.841 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:40 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:40.841 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:40 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:40.841 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:40 smithi181 ceph-mon[106466]: osdmap e668: 8 total, 8 up, 8 in 2024-03-29T05:04:40.841 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:40 smithi181 ceph-mon[101970]: failed to encode map e667 with expected crc 2024-03-29T05:04:40.841 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:40 smithi181 ceph-mon[101970]: failed to encode map e667 with expected crc 2024-03-29T05:04:40.841 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:40 smithi181 ceph-mon[101970]: failed to encode map e667 with expected crc 2024-03-29T05:04:40.841 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:40 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:40.842 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:40 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:40.842 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:40 smithi181 ceph-mon[101970]: osdmap e668: 8 total, 8 up, 8 in 2024-03-29T05:04:41.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:40 smithi187 ceph-mon[93536]: failed to encode map e667 with expected crc 2024-03-29T05:04:41.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:40 smithi187 ceph-mon[93536]: failed to encode map e667 with expected crc 2024-03-29T05:04:41.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:40 smithi187 ceph-mon[93536]: failed to encode map e667 with expected crc 2024-03-29T05:04:41.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:40 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:41.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:40 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:41.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:40 smithi187 ceph-mon[93536]: osdmap e668: 8 total, 8 up, 8 in 2024-03-29T05:04:42.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:41 smithi187 ceph-mon[93536]: pgmap v705: 521 pgs: 9 creating+peering, 19 unknown, 493 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 98 KiB/s rd, 7.5 KiB/s wr, 117 op/s 2024-03-29T05:04:42.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:41 smithi187 ceph-mon[93536]: failed to encode map e668 with expected crc 2024-03-29T05:04:42.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:41 smithi187 ceph-mon[93536]: failed to encode map e668 with expected crc 2024-03-29T05:04:42.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:41 smithi187 ceph-mon[93536]: failed to encode map e668 with expected crc 2024-03-29T05:04:42.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:42.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:42.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:04:42.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:04:42.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:04:42.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:42.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:41 smithi187 ceph-mon[93536]: osdmap e669: 8 total, 8 up, 8 in 2024-03-29T05:04:42.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[101970]: pgmap v705: 521 pgs: 9 creating+peering, 19 unknown, 493 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 98 KiB/s rd, 7.5 KiB/s wr, 117 op/s 2024-03-29T05:04:42.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[101970]: failed to encode map e668 with expected crc 2024-03-29T05:04:42.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[101970]: failed to encode map e668 with expected crc 2024-03-29T05:04:42.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[101970]: failed to encode map e668 with expected crc 2024-03-29T05:04:42.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:42.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:42.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:04:42.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:04:42.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:04:42.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:42.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[101970]: osdmap e669: 8 total, 8 up, 8 in 2024-03-29T05:04:42.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[106466]: pgmap v705: 521 pgs: 9 creating+peering, 19 unknown, 493 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 98 KiB/s rd, 7.5 KiB/s wr, 117 op/s 2024-03-29T05:04:42.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[106466]: failed to encode map e668 with expected crc 2024-03-29T05:04:42.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[106466]: failed to encode map e668 with expected crc 2024-03-29T05:04:42.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[106466]: failed to encode map e668 with expected crc 2024-03-29T05:04:42.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:42.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:42.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:04:42.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:04:42.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:04:42.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:42.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:41 smithi181 ceph-mon[106466]: osdmap e669: 8 total, 8 up, 8 in 2024-03-29T05:04:43.131 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.131 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.131 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.131 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.131 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.132 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 1 2024-03-29T05:04:43.132 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 1 2024-03-29T05:04:43.132 INFO:tasks.workunit.client.0.smithi181.stdout:child: image96 2024-03-29T05:04:43.132 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image96/adc64610dae4 2024-03-29T05:04:43.132 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.132 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.132 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.132 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.132 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.132 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.132 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.132 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.132 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.132 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.133 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.133 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.133 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 2 2024-03-29T05:04:43.133 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 2 2024-03-29T05:04:43.133 INFO:tasks.workunit.client.0.smithi181.stdout:child: image97 2024-03-29T05:04:43.133 INFO:tasks.workunit.client.0.smithi181.stdout:child: image96 2024-03-29T05:04:43.133 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image96/adc64610dae4 2024-03-29T05:04:43.133 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.133 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-27/image97/adc664f5e3e8 2024-03-29T05:04:43.133 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.133 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.133 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.133 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.133 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.134 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.134 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.134 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.134 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.134 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.134 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.134 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.134 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.134 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-28/image98 2024-03-29T05:04:43.134 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.134 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.134 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image98 2024-03-29T05:04:43.134 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image98 2024-03-29T05:04:43.134 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.134 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 3 2024-03-29T05:04:43.135 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 3 2024-03-29T05:04:43.135 INFO:tasks.workunit.client.0.smithi181.stdout:child: image97 2024-03-29T05:04:43.135 INFO:tasks.workunit.client.0.smithi181.stdout:child: image96 2024-03-29T05:04:43.135 INFO:tasks.workunit.client.0.smithi181.stdout:child: image98 2024-03-29T05:04:43.135 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image96/adc64610dae4 2024-03-29T05:04:43.135 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.135 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-27/image97/adc664f5e3e8 2024-03-29T05:04:43.135 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.135 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image98/adc68d0cfe5d 2024-03-29T05:04:43.135 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.135 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.135 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.135 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.135 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.136 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.136 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.136 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.136 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.136 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.136 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.136 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.136 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 3 2024-03-29T05:04:43.136 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 3 2024-03-29T05:04:43.136 INFO:tasks.workunit.client.0.smithi181.stdout:child: image97 2024-03-29T05:04:43.136 INFO:tasks.workunit.client.0.smithi181.stdout:child: image96 2024-03-29T05:04:43.136 INFO:tasks.workunit.client.0.smithi181.stdout:child: image98 2024-03-29T05:04:43.136 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image96/adc64610dae4 2024-03-29T05:04:43.136 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.136 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-27/image97/adc664f5e3e8 2024-03-29T05:04:43.137 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.137 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image98/adc68d0cfe5d 2024-03-29T05:04:43.137 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.137 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.137 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.137 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.137 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.137 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.137 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.137 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.137 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.137 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.137 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.138 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.138 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.138 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-28/image99 2024-03-29T05:04:43.138 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.138 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.138 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image99 2024-03-29T05:04:43.138 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image99 2024-03-29T05:04:43.138 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.138 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 4 2024-03-29T05:04:43.138 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 4 2024-03-29T05:04:43.139 INFO:tasks.workunit.client.0.smithi181.stdout:child: image97 2024-03-29T05:04:43.139 INFO:tasks.workunit.client.0.smithi181.stdout:child: image96 2024-03-29T05:04:43.139 INFO:tasks.workunit.client.0.smithi181.stdout:child: image98 2024-03-29T05:04:43.139 INFO:tasks.workunit.client.0.smithi181.stdout:child: image99 2024-03-29T05:04:43.139 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image96/adc64610dae4 2024-03-29T05:04:43.139 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.139 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-27/image97/adc664f5e3e8 2024-03-29T05:04:43.139 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.139 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image98/adc68d0cfe5d 2024-03-29T05:04:43.139 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.139 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image99/adc6e23a0884 2024-03-29T05:04:43.139 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.139 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.139 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.140 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.140 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.140 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.140 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.140 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.140 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.140 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.140 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.140 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.140 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.140 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-28/image98 2024-03-29T05:04:43.140 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.140 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.140 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image98 2024-03-29T05:04:43.140 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image98 2024-03-29T05:04:43.141 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.141 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.141 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-28/image99 2024-03-29T05:04:43.141 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.141 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image96 2024-03-29T05:04:43.141 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image98 2024-03-29T05:04:43.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:42 smithi187 ceph-mon[93536]: pgmap v707: 521 pgs: 9 creating+peering, 19 unknown, 493 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 2.6 KiB/s wr, 44 op/s 2024-03-29T05:04:43.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:42 smithi187 ceph-mon[93536]: pgmap v708: 521 pgs: 9 creating+peering, 19 unknown, 493 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:04:43.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:42 smithi187 ceph-mon[93536]: failed to encode map e669 with expected crc 2024-03-29T05:04:43.141 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:42 smithi187 ceph-mon[93536]: failed to encode map e669 with expected crc 2024-03-29T05:04:43.142 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:42 smithi187 ceph-mon[93536]: failed to encode map e669 with expected crc 2024-03-29T05:04:43.142 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:42 smithi187 ceph-mon[93536]: osdmap e670: 8 total, 8 up, 8 in 2024-03-29T05:04:43.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:42 smithi181 ceph-mon[101970]: pgmap v707: 521 pgs: 9 creating+peering, 19 unknown, 493 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 2.6 KiB/s wr, 44 op/s 2024-03-29T05:04:43.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:42 smithi181 ceph-mon[101970]: pgmap v708: 521 pgs: 9 creating+peering, 19 unknown, 493 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:04:43.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:42 smithi181 ceph-mon[101970]: failed to encode map e669 with expected crc 2024-03-29T05:04:43.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:42 smithi181 ceph-mon[101970]: failed to encode map e669 with expected crc 2024-03-29T05:04:43.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:42 smithi181 ceph-mon[101970]: failed to encode map e669 with expected crc 2024-03-29T05:04:43.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:42 smithi181 ceph-mon[101970]: osdmap e670: 8 total, 8 up, 8 in 2024-03-29T05:04:43.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:42 smithi181 ceph-mon[106466]: pgmap v707: 521 pgs: 9 creating+peering, 19 unknown, 493 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 2.6 KiB/s wr, 44 op/s 2024-03-29T05:04:43.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:42 smithi181 ceph-mon[106466]: pgmap v708: 521 pgs: 9 creating+peering, 19 unknown, 493 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:04:43.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:42 smithi181 ceph-mon[106466]: failed to encode map e669 with expected crc 2024-03-29T05:04:43.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:42 smithi181 ceph-mon[106466]: failed to encode map e669 with expected crc 2024-03-29T05:04:43.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:42 smithi181 ceph-mon[106466]: failed to encode map e669 with expected crc 2024-03-29T05:04:43.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:42 smithi181 ceph-mon[106466]: osdmap e670: 8 total, 8 up, 8 in 2024-03-29T05:04:43.937 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image99 2024-03-29T05:04:43.937 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image99 2024-03-29T05:04:43.937 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.937 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 4 2024-03-29T05:04:43.937 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 4 2024-03-29T05:04:43.937 INFO:tasks.workunit.client.0.smithi181.stdout:child: image97 2024-03-29T05:04:43.937 INFO:tasks.workunit.client.0.smithi181.stdout:child: image96 2024-03-29T05:04:43.937 INFO:tasks.workunit.client.0.smithi181.stdout:child: image98 2024-03-29T05:04:43.937 INFO:tasks.workunit.client.0.smithi181.stdout:child: image99 2024-03-29T05:04:43.938 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image96/adc64610dae4 2024-03-29T05:04:43.938 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.938 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-27/image97/adc664f5e3e8 2024-03-29T05:04:43.938 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.938 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image98/adc68d0cfe5d 2024-03-29T05:04:43.938 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.938 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image99/adc6e23a0884 2024-03-29T05:04:43.938 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.938 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.938 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.938 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.938 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.938 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.938 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.938 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-28/image98 2024-03-29T05:04:43.939 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.939 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image98 2024-03-29T05:04:43.939 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image98 2024-03-29T05:04:43.939 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.939 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.939 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-28/image99 2024-03-29T05:04:43.939 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.939 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image98 2024-03-29T05:04:43.939 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image99 2024-03-29T05:04:43.939 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image99 2024-03-29T05:04:43.939 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.939 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 3 2024-03-29T05:04:43.939 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 3 2024-03-29T05:04:43.939 INFO:tasks.workunit.client.0.smithi181.stdout:child: image97 2024-03-29T05:04:43.939 INFO:tasks.workunit.client.0.smithi181.stdout:child: image98 2024-03-29T05:04:43.940 INFO:tasks.workunit.client.0.smithi181.stdout:child: image99 2024-03-29T05:04:43.940 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-27/image97/adc664f5e3e8 2024-03-29T05:04:43.940 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.940 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image98/adc68d0cfe5d 2024-03-29T05:04:43.940 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.940 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image99/adc6e23a0884 2024-03-29T05:04:43.940 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.940 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.940 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.940 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.940 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.940 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.940 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.940 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-28/image99 2024-03-29T05:04:43.940 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.941 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-28/image99 2024-03-29T05:04:43.941 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image99 2024-03-29T05:04:43.941 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.941 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 2 2024-03-29T05:04:43.941 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 2 2024-03-29T05:04:43.941 INFO:tasks.workunit.client.0.smithi181.stdout:child: image97 2024-03-29T05:04:43.941 INFO:tasks.workunit.client.0.smithi181.stdout:child: image99 2024-03-29T05:04:43.941 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-27/image97/adc664f5e3e8 2024-03-29T05:04:43.941 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.941 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-28/image99/adc6e23a0884 2024-03-29T05:04:43.941 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.941 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.941 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.941 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.941 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-27/image97 2024-03-29T05:04:43.942 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.942 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 1 2024-03-29T05:04:43.942 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 1 2024-03-29T05:04:43.942 INFO:tasks.workunit.client.0.smithi181.stdout:child: image97 2024-03-29T05:04:43.942 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-27/image97/adc664f5e3e8 2024-03-29T05:04:43.942 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:43.942 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 0 2024-03-29T05:04:43.942 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 0 2024-03-29T05:04:43.942 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ListChildren (10059 ms) 2024-03-29T05:04:43.942 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ListChildrenTiered 2024-03-29T05:04:44.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:43 smithi187 ceph-mon[93536]: failed to encode map e670 with expected crc 2024-03-29T05:04:44.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:43 smithi187 ceph-mon[93536]: failed to encode map e670 with expected crc 2024-03-29T05:04:44.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:43 smithi187 ceph-mon[93536]: failed to encode map e670 with expected crc 2024-03-29T05:04:44.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:43 smithi181 ceph-mon[106466]: failed to encode map e670 with expected crc 2024-03-29T05:04:44.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:43 smithi181 ceph-mon[106466]: failed to encode map e670 with expected crc 2024-03-29T05:04:44.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:43 smithi181 ceph-mon[106466]: failed to encode map e670 with expected crc 2024-03-29T05:04:44.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:43 smithi181 ceph-mon[101970]: failed to encode map e670 with expected crc 2024-03-29T05:04:44.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:43 smithi181 ceph-mon[101970]: failed to encode map e670 with expected crc 2024-03-29T05:04:44.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:43 smithi181 ceph-mon[101970]: failed to encode map e670 with expected crc 2024-03-29T05:04:45.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:44 smithi187 ceph-mon[93536]: pgmap v711: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 5.6 KiB/s wr, 73 op/s 2024-03-29T05:04:45.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:44 smithi187 ceph-mon[93536]: osdmap e671: 8 total, 8 up, 8 in 2024-03-29T05:04:45.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:44 smithi187 ceph-mon[93536]: failed to encode map e671 with expected crc 2024-03-29T05:04:45.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:44 smithi187 ceph-mon[93536]: failed to encode map e671 with expected crc 2024-03-29T05:04:45.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:44 smithi187 ceph-mon[93536]: failed to encode map e671 with expected crc 2024-03-29T05:04:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:44 smithi181 ceph-mon[101970]: pgmap v711: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 5.6 KiB/s wr, 73 op/s 2024-03-29T05:04:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:44 smithi181 ceph-mon[101970]: osdmap e671: 8 total, 8 up, 8 in 2024-03-29T05:04:45.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:44 smithi181 ceph-mon[101970]: failed to encode map e671 with expected crc 2024-03-29T05:04:45.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:44 smithi181 ceph-mon[101970]: failed to encode map e671 with expected crc 2024-03-29T05:04:45.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:44 smithi181 ceph-mon[101970]: failed to encode map e671 with expected crc 2024-03-29T05:04:45.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:44 smithi181 ceph-mon[106466]: pgmap v711: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 5.6 KiB/s wr, 73 op/s 2024-03-29T05:04:45.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:44 smithi181 ceph-mon[106466]: osdmap e671: 8 total, 8 up, 8 in 2024-03-29T05:04:45.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:44 smithi181 ceph-mon[106466]: failed to encode map e671 with expected crc 2024-03-29T05:04:45.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:44 smithi181 ceph-mon[106466]: failed to encode map e671 with expected crc 2024-03-29T05:04:45.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:44 smithi181 ceph-mon[106466]: failed to encode map e671 with expected crc 2024-03-29T05:04:46.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:45 smithi187 ceph-mon[93536]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:45 smithi187 ceph-mon[93536]: osdmap e672: 8 total, 8 up, 8 in 2024-03-29T05:04:46.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:45 smithi187 ceph-mon[93536]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:45 smithi187 ceph-mon[93536]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:45 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1241726926' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:46.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:45 smithi187 ceph-mon[93536]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:45 smithi187 ceph-mon[93536]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:45 smithi187 ceph-mon[93536]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:45 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:46.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[106466]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[106466]: osdmap e672: 8 total, 8 up, 8 in 2024-03-29T05:04:46.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[106466]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[106466]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1241726926' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:46.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[106466]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[106466]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[106466]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:46.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[101970]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[101970]: osdmap e672: 8 total, 8 up, 8 in 2024-03-29T05:04:46.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[101970]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[101970]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1241726926' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:46.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[101970]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[101970]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[101970]: failed to encode map e672 with expected crc 2024-03-29T05:04:46.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:45 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:47.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:46 smithi187 ceph-mon[93536]: pgmap v714: 553 pgs: 32 unknown, 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 4.0 KiB/s wr, 52 op/s 2024-03-29T05:04:47.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:46 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:04:47.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:46 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:47.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:46 smithi187 ceph-mon[93536]: failed to encode map e673 with expected crc 2024-03-29T05:04:47.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:46 smithi187 ceph-mon[93536]: osdmap e673: 8 total, 8 up, 8 in 2024-03-29T05:04:47.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:46 smithi187 ceph-mon[93536]: failed to encode map e673 with expected crc 2024-03-29T05:04:47.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:46 smithi187 ceph-mon[93536]: failed to encode map e673 with expected crc 2024-03-29T05:04:47.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:46 smithi187 ceph-mon[93536]: failed to encode map e673 with expected crc 2024-03-29T05:04:47.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:46 smithi181 ceph-mon[101970]: pgmap v714: 553 pgs: 32 unknown, 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 4.0 KiB/s wr, 52 op/s 2024-03-29T05:04:47.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:46 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:04:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:46 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:46 smithi181 ceph-mon[101970]: failed to encode map e673 with expected crc 2024-03-29T05:04:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:46 smithi181 ceph-mon[101970]: osdmap e673: 8 total, 8 up, 8 in 2024-03-29T05:04:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:46 smithi181 ceph-mon[101970]: failed to encode map e673 with expected crc 2024-03-29T05:04:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:46 smithi181 ceph-mon[101970]: failed to encode map e673 with expected crc 2024-03-29T05:04:47.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:46 smithi181 ceph-mon[101970]: failed to encode map e673 with expected crc 2024-03-29T05:04:47.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:46 smithi181 ceph-mon[106466]: pgmap v714: 553 pgs: 32 unknown, 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 4.0 KiB/s wr, 52 op/s 2024-03-29T05:04:47.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:46 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:04:47.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:46 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:47.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:46 smithi181 ceph-mon[106466]: failed to encode map e673 with expected crc 2024-03-29T05:04:47.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:46 smithi181 ceph-mon[106466]: osdmap e673: 8 total, 8 up, 8 in 2024-03-29T05:04:47.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:46 smithi181 ceph-mon[106466]: failed to encode map e673 with expected crc 2024-03-29T05:04:47.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:46 smithi181 ceph-mon[106466]: failed to encode map e673 with expected crc 2024-03-29T05:04:47.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:46 smithi181 ceph-mon[106466]: failed to encode map e673 with expected crc 2024-03-29T05:04:48.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[106466]: failed to encode map e674 with expected crc 2024-03-29T05:04:48.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[106466]: failed to encode map e674 with expected crc 2024-03-29T05:04:48.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[106466]: osdmap e674: 8 total, 8 up, 8 in 2024-03-29T05:04:48.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[106466]: failed to encode map e674 with expected crc 2024-03-29T05:04:48.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/566602667' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:48.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[106466]: failed to encode map e674 with expected crc 2024-03-29T05:04:48.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[106466]: failed to encode map e674 with expected crc 2024-03-29T05:04:48.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/566602667' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:48.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[106466]: osdmap e675: 8 total, 8 up, 8 in 2024-03-29T05:04:48.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[101970]: failed to encode map e674 with expected crc 2024-03-29T05:04:48.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[101970]: failed to encode map e674 with expected crc 2024-03-29T05:04:48.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[101970]: osdmap e674: 8 total, 8 up, 8 in 2024-03-29T05:04:48.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[101970]: failed to encode map e674 with expected crc 2024-03-29T05:04:48.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/566602667' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:48.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[101970]: failed to encode map e674 with expected crc 2024-03-29T05:04:48.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[101970]: failed to encode map e674 with expected crc 2024-03-29T05:04:48.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/566602667' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:48.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:47 smithi181 ceph-mon[101970]: osdmap e675: 8 total, 8 up, 8 in 2024-03-29T05:04:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:47 smithi187 ceph-mon[93536]: failed to encode map e674 with expected crc 2024-03-29T05:04:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:47 smithi187 ceph-mon[93536]: failed to encode map e674 with expected crc 2024-03-29T05:04:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:47 smithi187 ceph-mon[93536]: osdmap e674: 8 total, 8 up, 8 in 2024-03-29T05:04:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:47 smithi187 ceph-mon[93536]: failed to encode map e674 with expected crc 2024-03-29T05:04:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:47 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/566602667' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:47 smithi187 ceph-mon[93536]: failed to encode map e674 with expected crc 2024-03-29T05:04:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:47 smithi187 ceph-mon[93536]: failed to encode map e674 with expected crc 2024-03-29T05:04:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:47 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/566602667' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:47 smithi187 ceph-mon[93536]: osdmap e675: 8 total, 8 up, 8 in 2024-03-29T05:04:49.205 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:04:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:04:48] "GET /metrics HTTP/1.1" 200 67637 "" "Prometheus/2.43.0" 2024-03-29T05:04:49.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:48 smithi181 ceph-mon[106466]: pgmap v717: 585 pgs: 8 creating+peering, 25 unknown, 552 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 549 KiB/s rd, 10 KiB/s wr, 664 op/s 2024-03-29T05:04:49.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:48 smithi181 ceph-mon[106466]: failed to encode map e675 with expected crc 2024-03-29T05:04:49.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:48 smithi181 ceph-mon[106466]: failed to encode map e675 with expected crc 2024-03-29T05:04:49.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:48 smithi181 ceph-mon[106466]: failed to encode map e675 with expected crc 2024-03-29T05:04:49.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:48 smithi181 ceph-mon[106466]: osdmap e676: 8 total, 8 up, 8 in 2024-03-29T05:04:49.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:48 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3017067700' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:49.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:48 smithi181 ceph-mon[101970]: pgmap v717: 585 pgs: 8 creating+peering, 25 unknown, 552 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 549 KiB/s rd, 10 KiB/s wr, 664 op/s 2024-03-29T05:04:49.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:48 smithi181 ceph-mon[101970]: failed to encode map e675 with expected crc 2024-03-29T05:04:49.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:48 smithi181 ceph-mon[101970]: failed to encode map e675 with expected crc 2024-03-29T05:04:49.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:48 smithi181 ceph-mon[101970]: failed to encode map e675 with expected crc 2024-03-29T05:04:49.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:48 smithi181 ceph-mon[101970]: osdmap e676: 8 total, 8 up, 8 in 2024-03-29T05:04:49.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:48 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3017067700' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:49.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:48 smithi187 ceph-mon[93536]: pgmap v717: 585 pgs: 8 creating+peering, 25 unknown, 552 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 549 KiB/s rd, 10 KiB/s wr, 664 op/s 2024-03-29T05:04:49.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:48 smithi187 ceph-mon[93536]: failed to encode map e675 with expected crc 2024-03-29T05:04:49.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:48 smithi187 ceph-mon[93536]: failed to encode map e675 with expected crc 2024-03-29T05:04:49.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:48 smithi187 ceph-mon[93536]: failed to encode map e675 with expected crc 2024-03-29T05:04:49.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:48 smithi187 ceph-mon[93536]: osdmap e676: 8 total, 8 up, 8 in 2024-03-29T05:04:49.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:48 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3017067700' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:04:50.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:49 smithi181 ceph-mon[106466]: failed to encode map e676 with expected crc 2024-03-29T05:04:50.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:49 smithi181 ceph-mon[106466]: failed to encode map e676 with expected crc 2024-03-29T05:04:50.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:49 smithi181 ceph-mon[106466]: failed to encode map e676 with expected crc 2024-03-29T05:04:50.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:49 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3017067700' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:50.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:49 smithi181 ceph-mon[106466]: osdmap e677: 8 total, 8 up, 8 in 2024-03-29T05:04:50.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:49 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31", "force_nonempty":""}]: dispatch 2024-03-29T05:04:50.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:49 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31", "force_nonempty":""}]: dispatch 2024-03-29T05:04:50.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:49 smithi181 ceph-mon[101970]: failed to encode map e676 with expected crc 2024-03-29T05:04:50.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:49 smithi181 ceph-mon[101970]: failed to encode map e676 with expected crc 2024-03-29T05:04:50.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:49 smithi181 ceph-mon[101970]: failed to encode map e676 with expected crc 2024-03-29T05:04:50.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:49 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3017067700' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:50.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:49 smithi181 ceph-mon[101970]: osdmap e677: 8 total, 8 up, 8 in 2024-03-29T05:04:50.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:49 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31", "force_nonempty":""}]: dispatch 2024-03-29T05:04:50.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:49 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31", "force_nonempty":""}]: dispatch 2024-03-29T05:04:50.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:49 smithi187 ceph-mon[93536]: failed to encode map e676 with expected crc 2024-03-29T05:04:50.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:49 smithi187 ceph-mon[93536]: failed to encode map e676 with expected crc 2024-03-29T05:04:50.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:49 smithi187 ceph-mon[93536]: failed to encode map e676 with expected crc 2024-03-29T05:04:50.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:49 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3017067700' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:04:50.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:49 smithi187 ceph-mon[93536]: osdmap e677: 8 total, 8 up, 8 in 2024-03-29T05:04:50.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:49 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31", "force_nonempty":""}]: dispatch 2024-03-29T05:04:50.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:49 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31", "force_nonempty":""}]: dispatch 2024-03-29T05:04:51.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[106466]: pgmap v720: 617 pgs: 8 creating+peering, 50 unknown, 559 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 556 KiB/s rd, 12 KiB/s wr, 677 op/s 2024-03-29T05:04:51.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[106466]: failed to encode map e677 with expected crc 2024-03-29T05:04:51.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[106466]: failed to encode map e677 with expected crc 2024-03-29T05:04:51.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[106466]: failed to encode map e677 with expected crc 2024-03-29T05:04:51.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[106466]: failed to encode map e677 with expected crc 2024-03-29T05:04:51.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31", "force_nonempty":""}]': finished 2024-03-29T05:04:51.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[106466]: osdmap e678: 8 total, 8 up, 8 in 2024-03-29T05:04:51.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi181-86707-31", "mode":"writeback"}]: dispatch 2024-03-29T05:04:51.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi181-86707-31", "mode":"writeback"}]: dispatch 2024-03-29T05:04:51.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[101970]: pgmap v720: 617 pgs: 8 creating+peering, 50 unknown, 559 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 556 KiB/s rd, 12 KiB/s wr, 677 op/s 2024-03-29T05:04:51.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[101970]: failed to encode map e677 with expected crc 2024-03-29T05:04:51.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[101970]: failed to encode map e677 with expected crc 2024-03-29T05:04:51.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[101970]: failed to encode map e677 with expected crc 2024-03-29T05:04:51.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[101970]: failed to encode map e677 with expected crc 2024-03-29T05:04:51.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31", "force_nonempty":""}]': finished 2024-03-29T05:04:51.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[101970]: osdmap e678: 8 total, 8 up, 8 in 2024-03-29T05:04:51.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi181-86707-31", "mode":"writeback"}]: dispatch 2024-03-29T05:04:51.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:50 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi181-86707-31", "mode":"writeback"}]: dispatch 2024-03-29T05:04:51.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:50 smithi187 ceph-mon[93536]: pgmap v720: 617 pgs: 8 creating+peering, 50 unknown, 559 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 556 KiB/s rd, 12 KiB/s wr, 677 op/s 2024-03-29T05:04:51.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:50 smithi187 ceph-mon[93536]: failed to encode map e677 with expected crc 2024-03-29T05:04:51.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:50 smithi187 ceph-mon[93536]: failed to encode map e677 with expected crc 2024-03-29T05:04:51.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:50 smithi187 ceph-mon[93536]: failed to encode map e677 with expected crc 2024-03-29T05:04:51.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:50 smithi187 ceph-mon[93536]: failed to encode map e677 with expected crc 2024-03-29T05:04:51.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:50 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31", "force_nonempty":""}]': finished 2024-03-29T05:04:51.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:50 smithi187 ceph-mon[93536]: osdmap e678: 8 total, 8 up, 8 in 2024-03-29T05:04:51.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:50 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi181-86707-31", "mode":"writeback"}]: dispatch 2024-03-29T05:04:51.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:50 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi181-86707-31", "mode":"writeback"}]: dispatch 2024-03-29T05:04:52.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:51 smithi181 ceph-mon[101970]: failed to encode map e678 with expected crc 2024-03-29T05:04:52.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:51 smithi181 ceph-mon[101970]: failed to encode map e678 with expected crc 2024-03-29T05:04:52.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:51 smithi181 ceph-mon[101970]: Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-03-29T05:04:52.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:51 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi181-86707-31", "mode":"writeback"}]': finished 2024-03-29T05:04:52.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:51 smithi181 ceph-mon[101970]: osdmap e679: 8 total, 8 up, 8 in 2024-03-29T05:04:52.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:51 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi181-86707-29", "overlaypool":"test-librbd-smithi181-86707-31"}]: dispatch 2024-03-29T05:04:52.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:51 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi181-86707-29", "overlaypool":"test-librbd-smithi181-86707-31"}]: dispatch 2024-03-29T05:04:52.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:51 smithi181 ceph-mon[106466]: failed to encode map e678 with expected crc 2024-03-29T05:04:52.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:51 smithi181 ceph-mon[106466]: failed to encode map e678 with expected crc 2024-03-29T05:04:52.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:51 smithi181 ceph-mon[106466]: Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-03-29T05:04:52.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:51 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi181-86707-31", "mode":"writeback"}]': finished 2024-03-29T05:04:52.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:51 smithi181 ceph-mon[106466]: osdmap e679: 8 total, 8 up, 8 in 2024-03-29T05:04:52.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:51 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi181-86707-29", "overlaypool":"test-librbd-smithi181-86707-31"}]: dispatch 2024-03-29T05:04:52.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:51 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi181-86707-29", "overlaypool":"test-librbd-smithi181-86707-31"}]: dispatch 2024-03-29T05:04:52.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:51 smithi187 ceph-mon[93536]: failed to encode map e678 with expected crc 2024-03-29T05:04:52.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:51 smithi187 ceph-mon[93536]: failed to encode map e678 with expected crc 2024-03-29T05:04:52.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:51 smithi187 ceph-mon[93536]: Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-03-29T05:04:52.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:51 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi181-86707-31", "mode":"writeback"}]': finished 2024-03-29T05:04:52.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:51 smithi187 ceph-mon[93536]: osdmap e679: 8 total, 8 up, 8 in 2024-03-29T05:04:52.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:51 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi181-86707-29", "overlaypool":"test-librbd-smithi181-86707-31"}]: dispatch 2024-03-29T05:04:52.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:51 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi181-86707-29", "overlaypool":"test-librbd-smithi181-86707-31"}]: dispatch 2024-03-29T05:04:52.648 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:04:53.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:52 smithi181 ceph-mon[101970]: failed to encode map e678 with expected crc 2024-03-29T05:04:53.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:52 smithi181 ceph-mon[101970]: pgmap v723: 617 pgs: 8 creating+peering, 50 unknown, 559 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 1.7 KiB/s wr, 12 op/s 2024-03-29T05:04:53.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:52 smithi181 ceph-mon[101970]: failed to encode map e679 with expected crc 2024-03-29T05:04:53.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:52 smithi181 ceph-mon[101970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:04:53.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:52 smithi181 ceph-mon[101970]: failed to encode map e679 with expected crc 2024-03-29T05:04:53.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:52 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi181-86707-29", "overlaypool":"test-librbd-smithi181-86707-31"}]': finished 2024-03-29T05:04:53.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:52 smithi181 ceph-mon[101970]: osdmap e680: 8 total, 8 up, 8 in 2024-03-29T05:04:53.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:52 smithi181 ceph-mon[106466]: failed to encode map e678 with expected crc 2024-03-29T05:04:53.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:52 smithi181 ceph-mon[106466]: pgmap v723: 617 pgs: 8 creating+peering, 50 unknown, 559 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 1.7 KiB/s wr, 12 op/s 2024-03-29T05:04:53.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:52 smithi181 ceph-mon[106466]: failed to encode map e679 with expected crc 2024-03-29T05:04:53.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:52 smithi181 ceph-mon[106466]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:04:53.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:52 smithi181 ceph-mon[106466]: failed to encode map e679 with expected crc 2024-03-29T05:04:53.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:52 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi181-86707-29", "overlaypool":"test-librbd-smithi181-86707-31"}]': finished 2024-03-29T05:04:53.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:52 smithi181 ceph-mon[106466]: osdmap e680: 8 total, 8 up, 8 in 2024-03-29T05:04:53.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:52 smithi187 ceph-mon[93536]: failed to encode map e678 with expected crc 2024-03-29T05:04:53.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:52 smithi187 ceph-mon[93536]: pgmap v723: 617 pgs: 8 creating+peering, 50 unknown, 559 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 1.7 KiB/s wr, 12 op/s 2024-03-29T05:04:53.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:52 smithi187 ceph-mon[93536]: failed to encode map e679 with expected crc 2024-03-29T05:04:53.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:52 smithi187 ceph-mon[93536]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:04:53.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:52 smithi187 ceph-mon[93536]: failed to encode map e679 with expected crc 2024-03-29T05:04:53.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:52 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi181-86707-29", "overlaypool":"test-librbd-smithi181-86707-31"}]': finished 2024-03-29T05:04:53.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:52 smithi187 ceph-mon[93536]: osdmap e680: 8 total, 8 up, 8 in 2024-03-29T05:04:54.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:53 smithi181 ceph-mon[101970]: failed to encode map e679 with expected crc 2024-03-29T05:04:54.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:53 smithi181 ceph-mon[101970]: failed to encode map e680 with expected crc 2024-03-29T05:04:54.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:53 smithi181 ceph-mon[101970]: failed to encode map e680 with expected crc 2024-03-29T05:04:54.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:53 smithi181 ceph-mon[101970]: failed to encode map e680 with expected crc 2024-03-29T05:04:54.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:53 smithi181 ceph-mon[101970]: osdmap e681: 8 total, 8 up, 8 in 2024-03-29T05:04:54.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:53 smithi181 ceph-mon[106466]: failed to encode map e679 with expected crc 2024-03-29T05:04:54.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:53 smithi181 ceph-mon[106466]: failed to encode map e680 with expected crc 2024-03-29T05:04:54.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:53 smithi181 ceph-mon[106466]: failed to encode map e680 with expected crc 2024-03-29T05:04:54.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:53 smithi181 ceph-mon[106466]: failed to encode map e680 with expected crc 2024-03-29T05:04:54.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:53 smithi181 ceph-mon[106466]: osdmap e681: 8 total, 8 up, 8 in 2024-03-29T05:04:54.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:53 smithi187 ceph-mon[93536]: failed to encode map e679 with expected crc 2024-03-29T05:04:54.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:53 smithi187 ceph-mon[93536]: failed to encode map e680 with expected crc 2024-03-29T05:04:54.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:53 smithi187 ceph-mon[93536]: failed to encode map e680 with expected crc 2024-03-29T05:04:54.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:53 smithi187 ceph-mon[93536]: failed to encode map e680 with expected crc 2024-03-29T05:04:54.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:53 smithi187 ceph-mon[93536]: osdmap e681: 8 total, 8 up, 8 in 2024-03-29T05:04:55.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:54 smithi181 ceph-mon[106466]: pgmap v726: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:04:55.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:54 smithi181 ceph-mon[106466]: failed to encode map e681 with expected crc 2024-03-29T05:04:55.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:54 smithi181 ceph-mon[106466]: failed to encode map e681 with expected crc 2024-03-29T05:04:55.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:54 smithi181 ceph-mon[106466]: osdmap e682: 8 total, 8 up, 8 in 2024-03-29T05:04:55.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:54 smithi181 ceph-mon[101970]: pgmap v726: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:04:55.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:54 smithi181 ceph-mon[101970]: failed to encode map e681 with expected crc 2024-03-29T05:04:55.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:54 smithi181 ceph-mon[101970]: failed to encode map e681 with expected crc 2024-03-29T05:04:55.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:54 smithi181 ceph-mon[101970]: osdmap e682: 8 total, 8 up, 8 in 2024-03-29T05:04:55.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:54 smithi187 ceph-mon[93536]: pgmap v726: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:04:55.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:54 smithi187 ceph-mon[93536]: failed to encode map e681 with expected crc 2024-03-29T05:04:55.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:54 smithi187 ceph-mon[93536]: failed to encode map e681 with expected crc 2024-03-29T05:04:55.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:54 smithi187 ceph-mon[93536]: osdmap e682: 8 total, 8 up, 8 in 2024-03-29T05:04:56.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:55 smithi187 ceph-mon[93536]: failed to encode map e681 with expected crc 2024-03-29T05:04:56.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:55 smithi187 ceph-mon[93536]: failed to encode map e682 with expected crc 2024-03-29T05:04:56.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:55 smithi187 ceph-mon[93536]: failed to encode map e682 with expected crc 2024-03-29T05:04:56.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:55 smithi187 ceph-mon[93536]: failed to encode map e682 with expected crc 2024-03-29T05:04:56.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:55 smithi187 ceph-mon[93536]: osdmap e683: 8 total, 8 up, 8 in 2024-03-29T05:04:56.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:55 smithi181 ceph-mon[106466]: failed to encode map e681 with expected crc 2024-03-29T05:04:56.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:55 smithi181 ceph-mon[106466]: failed to encode map e682 with expected crc 2024-03-29T05:04:56.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:55 smithi181 ceph-mon[106466]: failed to encode map e682 with expected crc 2024-03-29T05:04:56.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:55 smithi181 ceph-mon[106466]: failed to encode map e682 with expected crc 2024-03-29T05:04:56.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:55 smithi181 ceph-mon[106466]: osdmap e683: 8 total, 8 up, 8 in 2024-03-29T05:04:56.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:55 smithi181 ceph-mon[101970]: failed to encode map e681 with expected crc 2024-03-29T05:04:56.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:55 smithi181 ceph-mon[101970]: failed to encode map e682 with expected crc 2024-03-29T05:04:56.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:55 smithi181 ceph-mon[101970]: failed to encode map e682 with expected crc 2024-03-29T05:04:56.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:55 smithi181 ceph-mon[101970]: failed to encode map e682 with expected crc 2024-03-29T05:04:56.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:55 smithi181 ceph-mon[101970]: osdmap e683: 8 total, 8 up, 8 in 2024-03-29T05:04:57.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:56 smithi187 ceph-mon[93536]: pgmap v729: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:04:57.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:56 smithi187 ceph-mon[93536]: failed to encode map e683 with expected crc 2024-03-29T05:04:57.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:56 smithi187 ceph-mon[93536]: failed to encode map e683 with expected crc 2024-03-29T05:04:57.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:56 smithi187 ceph-mon[93536]: failed to encode map e683 with expected crc 2024-03-29T05:04:57.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:57.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:04:57.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:56 smithi187 ceph-mon[93536]: osdmap e684: 8 total, 8 up, 8 in 2024-03-29T05:04:57.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:56 smithi181 ceph-mon[106466]: pgmap v729: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:04:57.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:56 smithi181 ceph-mon[106466]: failed to encode map e683 with expected crc 2024-03-29T05:04:57.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:56 smithi181 ceph-mon[106466]: failed to encode map e683 with expected crc 2024-03-29T05:04:57.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:56 smithi181 ceph-mon[106466]: failed to encode map e683 with expected crc 2024-03-29T05:04:57.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:57.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:04:57.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:56 smithi181 ceph-mon[106466]: osdmap e684: 8 total, 8 up, 8 in 2024-03-29T05:04:57.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:56 smithi181 ceph-mon[101970]: pgmap v729: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:04:57.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:56 smithi181 ceph-mon[101970]: failed to encode map e683 with expected crc 2024-03-29T05:04:57.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:56 smithi181 ceph-mon[101970]: failed to encode map e683 with expected crc 2024-03-29T05:04:57.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:56 smithi181 ceph-mon[101970]: failed to encode map e683 with expected crc 2024-03-29T05:04:57.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:04:57.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:04:57.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:56 smithi181 ceph-mon[101970]: osdmap e684: 8 total, 8 up, 8 in 2024-03-29T05:04:58.192 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.192 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.192 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.192 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.192 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.192 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 1 2024-03-29T05:04:58.192 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 1 2024-03-29T05:04:58.192 INFO:tasks.workunit.client.0.smithi181.stdout:child: image101 2024-03-29T05:04:58.192 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image101/adc6c71e76fc 2024-03-29T05:04:58.192 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.192 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.193 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.193 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.193 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.193 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.193 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.193 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.193 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.193 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.193 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.193 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.193 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 2 2024-03-29T05:04:58.193 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 2 2024-03-29T05:04:58.193 INFO:tasks.workunit.client.0.smithi181.stdout:child: image102 2024-03-29T05:04:58.193 INFO:tasks.workunit.client.0.smithi181.stdout:child: image101 2024-03-29T05:04:58.194 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image101/adc6c71e76fc 2024-03-29T05:04:58.194 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.194 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-29/image102/adc642d71d74 2024-03-29T05:04:58.194 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.194 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.194 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.194 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.194 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.194 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.194 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.194 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.194 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.194 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.194 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.195 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.195 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.195 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-30/image103 2024-03-29T05:04:58.195 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.195 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.195 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image103 2024-03-29T05:04:58.195 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image103 2024-03-29T05:04:58.195 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.195 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 3 2024-03-29T05:04:58.195 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 3 2024-03-29T05:04:58.195 INFO:tasks.workunit.client.0.smithi181.stdout:child: image102 2024-03-29T05:04:58.195 INFO:tasks.workunit.client.0.smithi181.stdout:child: image101 2024-03-29T05:04:58.195 INFO:tasks.workunit.client.0.smithi181.stdout:child: image103 2024-03-29T05:04:58.195 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image101/adc6c71e76fc 2024-03-29T05:04:58.196 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.196 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-29/image102/adc642d71d74 2024-03-29T05:04:58.196 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.196 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image103/adc657771428 2024-03-29T05:04:58.196 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.196 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.196 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.196 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.196 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.196 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.196 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.196 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.196 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.196 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.196 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.197 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.197 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 3 2024-03-29T05:04:58.197 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 3 2024-03-29T05:04:58.197 INFO:tasks.workunit.client.0.smithi181.stdout:child: image102 2024-03-29T05:04:58.197 INFO:tasks.workunit.client.0.smithi181.stdout:child: image101 2024-03-29T05:04:58.197 INFO:tasks.workunit.client.0.smithi181.stdout:child: image103 2024-03-29T05:04:58.197 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image101/adc6c71e76fc 2024-03-29T05:04:58.197 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.197 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-29/image102/adc642d71d74 2024-03-29T05:04:58.197 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.197 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image103/adc657771428 2024-03-29T05:04:58.197 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.197 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.197 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.197 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.198 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.198 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.198 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.198 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.198 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.198 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.198 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.198 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.198 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.198 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-30/image104 2024-03-29T05:04:58.198 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.198 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.198 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image104 2024-03-29T05:04:58.198 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image104 2024-03-29T05:04:58.199 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.199 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 4 2024-03-29T05:04:58.199 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 4 2024-03-29T05:04:58.199 INFO:tasks.workunit.client.0.smithi181.stdout:child: image102 2024-03-29T05:04:58.199 INFO:tasks.workunit.client.0.smithi181.stdout:child: image101 2024-03-29T05:04:58.199 INFO:tasks.workunit.client.0.smithi181.stdout:child: image103 2024-03-29T05:04:58.199 INFO:tasks.workunit.client.0.smithi181.stdout:child: image104 2024-03-29T05:04:58.199 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image101/adc6c71e76fc 2024-03-29T05:04:58.199 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.199 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-29/image102/adc642d71d74 2024-03-29T05:04:58.199 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.199 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image103/adc657771428 2024-03-29T05:04:58.199 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.199 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image104/adc6aaa74717 2024-03-29T05:04:58.200 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.200 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.200 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.200 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.200 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.200 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.200 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.200 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.200 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.200 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.200 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.200 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.200 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.200 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-30/image103 2024-03-29T05:04:58.201 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.201 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image101 2024-03-29T05:04:58.201 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image103 2024-03-29T05:04:58.201 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image103 2024-03-29T05:04:58.201 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.201 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:04:58.201 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-30/image104 2024-03-29T05:04:58.201 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:04:58.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:57 smithi187 ceph-mon[93536]: failed to encode map e684 with expected crc 2024-03-29T05:04:58.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:57 smithi187 ceph-mon[93536]: failed to encode map e684 with expected crc 2024-03-29T05:04:58.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:57 smithi187 ceph-mon[93536]: osdmap e685: 8 total, 8 up, 8 in 2024-03-29T05:04:58.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:57 smithi181 ceph-mon[101970]: failed to encode map e684 with expected crc 2024-03-29T05:04:58.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:57 smithi181 ceph-mon[101970]: failed to encode map e684 with expected crc 2024-03-29T05:04:58.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:57 smithi181 ceph-mon[101970]: osdmap e685: 8 total, 8 up, 8 in 2024-03-29T05:04:58.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:57 smithi181 ceph-mon[106466]: failed to encode map e684 with expected crc 2024-03-29T05:04:58.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:57 smithi181 ceph-mon[106466]: failed to encode map e684 with expected crc 2024-03-29T05:04:58.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:57 smithi181 ceph-mon[106466]: osdmap e685: 8 total, 8 up, 8 in 2024-03-29T05:04:59.205 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:04:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:04:58] "GET /metrics HTTP/1.1" 200 74884 "" "Prometheus/2.43.0" 2024-03-29T05:04:59.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:58 smithi181 ceph-mon[106466]: pgmap v732: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.2 KiB/s wr, 37 op/s 2024-03-29T05:04:59.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:58 smithi181 ceph-mon[106466]: failed to encode map e685 with expected crc 2024-03-29T05:04:59.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:58 smithi181 ceph-mon[106466]: failed to encode map e685 with expected crc 2024-03-29T05:04:59.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:58 smithi181 ceph-mon[101970]: pgmap v732: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.2 KiB/s wr, 37 op/s 2024-03-29T05:04:59.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:58 smithi181 ceph-mon[101970]: failed to encode map e685 with expected crc 2024-03-29T05:04:59.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:58 smithi181 ceph-mon[101970]: failed to encode map e685 with expected crc 2024-03-29T05:04:59.287 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:58 smithi187 ceph-mon[93536]: pgmap v732: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.2 KiB/s wr, 37 op/s 2024-03-29T05:04:59.287 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:58 smithi187 ceph-mon[93536]: failed to encode map e685 with expected crc 2024-03-29T05:04:59.287 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:58 smithi187 ceph-mon[93536]: failed to encode map e685 with expected crc 2024-03-29T05:05:00.306 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:59 smithi187 ceph-mon[93536]: osdmap e686: 8 total, 8 up, 8 in 2024-03-29T05:05:00.306 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:59 smithi187 ceph-mon[93536]: failed to encode map e686 with expected crc 2024-03-29T05:05:00.306 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:59 smithi187 ceph-mon[93536]: failed to encode map e686 with expected crc 2024-03-29T05:05:00.306 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:59 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi181-86707-29"}]: dispatch 2024-03-29T05:05:00.307 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:04:59 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi181-86707-29"}]: dispatch 2024-03-29T05:05:00.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:59 smithi181 ceph-mon[106466]: osdmap e686: 8 total, 8 up, 8 in 2024-03-29T05:05:00.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:59 smithi181 ceph-mon[106466]: failed to encode map e686 with expected crc 2024-03-29T05:05:00.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:59 smithi181 ceph-mon[106466]: failed to encode map e686 with expected crc 2024-03-29T05:05:00.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:59 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi181-86707-29"}]: dispatch 2024-03-29T05:05:00.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:04:59 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi181-86707-29"}]: dispatch 2024-03-29T05:05:00.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:59 smithi181 ceph-mon[101970]: osdmap e686: 8 total, 8 up, 8 in 2024-03-29T05:05:00.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:59 smithi181 ceph-mon[101970]: failed to encode map e686 with expected crc 2024-03-29T05:05:00.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:59 smithi181 ceph-mon[101970]: failed to encode map e686 with expected crc 2024-03-29T05:05:00.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:59 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi181-86707-29"}]: dispatch 2024-03-29T05:05:00.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:04:59 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi181-86707-29"}]: dispatch 2024-03-29T05:05:00.985 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image101 2024-03-29T05:05:00.985 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image103 2024-03-29T05:05:00.985 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image104 2024-03-29T05:05:00.986 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image104 2024-03-29T05:05:00.986 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.986 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 4 2024-03-29T05:05:00.986 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 4 2024-03-29T05:05:00.986 INFO:tasks.workunit.client.0.smithi181.stdout:child: image102 2024-03-29T05:05:00.986 INFO:tasks.workunit.client.0.smithi181.stdout:child: image101 2024-03-29T05:05:00.986 INFO:tasks.workunit.client.0.smithi181.stdout:child: image103 2024-03-29T05:05:00.986 INFO:tasks.workunit.client.0.smithi181.stdout:child: image104 2024-03-29T05:05:00.986 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image101/adc6c71e76fc 2024-03-29T05:05:00.986 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.986 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-29/image102/adc642d71d74 2024-03-29T05:05:00.986 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.986 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image103/adc657771428 2024-03-29T05:05:00.987 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.987 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image104/adc6aaa74717 2024-03-29T05:05:00.987 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.987 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.987 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-29/image102 2024-03-29T05:05:00.987 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:05:00.987 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-29/image102 2024-03-29T05:05:00.987 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.987 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.987 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-30/image103 2024-03-29T05:05:00.987 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:05:00.987 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image103 2024-03-29T05:05:00.987 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image103 2024-03-29T05:05:00.987 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.988 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.988 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-30/image104 2024-03-29T05:05:00.988 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:05:00.988 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image103 2024-03-29T05:05:00.988 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image104 2024-03-29T05:05:00.988 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image104 2024-03-29T05:05:00.988 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.988 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 3 2024-03-29T05:05:00.988 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 3 2024-03-29T05:05:00.988 INFO:tasks.workunit.client.0.smithi181.stdout:child: image102 2024-03-29T05:05:00.988 INFO:tasks.workunit.client.0.smithi181.stdout:child: image103 2024-03-29T05:05:00.988 INFO:tasks.workunit.client.0.smithi181.stdout:child: image104 2024-03-29T05:05:00.988 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-29/image102/adc642d71d74 2024-03-29T05:05:00.988 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.989 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image103/adc657771428 2024-03-29T05:05:00.989 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.989 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image104/adc6aaa74717 2024-03-29T05:05:00.989 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.989 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.989 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-29/image102 2024-03-29T05:05:00.989 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:05:00.989 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-29/image102 2024-03-29T05:05:00.989 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.989 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.989 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-30/image104 2024-03-29T05:05:00.989 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:05:00.989 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-30/image104 2024-03-29T05:05:00.989 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image104 2024-03-29T05:05:00.989 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.990 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 2 2024-03-29T05:05:00.990 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 2 2024-03-29T05:05:00.990 INFO:tasks.workunit.client.0.smithi181.stdout:child: image102 2024-03-29T05:05:00.990 INFO:tasks.workunit.client.0.smithi181.stdout:child: image104 2024-03-29T05:05:00.990 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-29/image102/adc642d71d74 2024-03-29T05:05:00.990 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.990 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-30/image104/adc6aaa74717 2024-03-29T05:05:00.990 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.990 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.990 INFO:tasks.workunit.client.0.smithi181.stdout:trying to find test-librbd-smithi181-86707-29/image102 2024-03-29T05:05:00.990 INFO:tasks.workunit.client.0.smithi181.stdout:checking test-librbd-smithi181-86707-29/image102 2024-03-29T05:05:00.990 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-29/image102 2024-03-29T05:05:00.990 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.990 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 1 2024-03-29T05:05:00.991 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 1 2024-03-29T05:05:00.991 INFO:tasks.workunit.client.0.smithi181.stdout:child: image102 2024-03-29T05:05:00.991 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-29/image102/adc642d71d74 2024-03-29T05:05:00.991 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:05:00.991 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 0 2024-03-29T05:05:00.991 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 0 2024-03-29T05:05:00.991 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ListChildrenTiered (17048 ms) 2024-03-29T05:05:00.991 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.LockingPP 2024-03-29T05:05:00.991 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:01.041 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.LockingPP (56 ms) 2024-03-29T05:05:01.041 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.FlushAio 2024-03-29T05:05:01.041 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:01.281 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.FlushAio (240 ms) 2024-03-29T05:05:01.281 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.FlushAioPP 2024-03-29T05:05:01.281 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:01.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:00 smithi187 ceph-mon[93536]: pgmap v735: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 3.7 KiB/s wr, 98 op/s 2024-03-29T05:05:01.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:00 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi181-86707-29"}]': finished 2024-03-29T05:05:01.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:00 smithi187 ceph-mon[93536]: failed to encode map e687 with expected crc 2024-03-29T05:05:01.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:00 smithi187 ceph-mon[93536]: osdmap e687: 8 total, 8 up, 8 in 2024-03-29T05:05:01.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:00 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31"}]: dispatch 2024-03-29T05:05:01.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:00 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31"}]: dispatch 2024-03-29T05:05:01.435 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.FlushAioPP (154 ms) 2024-03-29T05:05:01.435 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ZeroLengthWrite 2024-03-29T05:05:01.435 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:01.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:00 smithi181 ceph-mon[101970]: pgmap v735: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 3.7 KiB/s wr, 98 op/s 2024-03-29T05:05:01.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:00 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi181-86707-29"}]': finished 2024-03-29T05:05:01.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:00 smithi181 ceph-mon[101970]: failed to encode map e687 with expected crc 2024-03-29T05:05:01.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:00 smithi181 ceph-mon[101970]: osdmap e687: 8 total, 8 up, 8 in 2024-03-29T05:05:01.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:00 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31"}]: dispatch 2024-03-29T05:05:01.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:00 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31"}]: dispatch 2024-03-29T05:05:01.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:00 smithi181 ceph-mon[106466]: pgmap v735: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 3.7 KiB/s wr, 98 op/s 2024-03-29T05:05:01.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:00 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi181-86707-29"}]': finished 2024-03-29T05:05:01.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:00 smithi181 ceph-mon[106466]: failed to encode map e687 with expected crc 2024-03-29T05:05:01.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:00 smithi181 ceph-mon[106466]: osdmap e687: 8 total, 8 up, 8 in 2024-03-29T05:05:01.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:00 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31"}]: dispatch 2024-03-29T05:05:01.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:00 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31"}]: dispatch 2024-03-29T05:05:01.459 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ZeroLengthWrite (24 ms) 2024-03-29T05:05:01.459 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ZeroLengthDiscard 2024-03-29T05:05:01.459 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:01.483 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ZeroLengthDiscard (24 ms) 2024-03-29T05:05:01.483 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ZeroLengthRead 2024-03-29T05:05:01.483 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:01.498 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ZeroLengthRead (15 ms) 2024-03-29T05:05:01.498 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.LargeCacheRead 2024-03-29T05:05:01.499 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:01.536 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.LargeCacheRead (38 ms) 2024-03-29T05:05:01.536 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestPendingAio 2024-03-29T05:05:01.536 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:01.735 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestPendingAio (199 ms) 2024-03-29T05:05:01.735 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyup 2024-03-29T05:05:01.735 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:02.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:01 smithi187 ceph-mon[93536]: failed to encode map e687 with expected crc 2024-03-29T05:05:02.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:01 smithi187 ceph-mon[93536]: failed to encode map e687 with expected crc 2024-03-29T05:05:02.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:01 smithi187 ceph-mon[93536]: Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-03-29T05:05:02.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:01 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31"}]': finished 2024-03-29T05:05:02.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:01 smithi187 ceph-mon[93536]: failed to encode map e688 with expected crc 2024-03-29T05:05:02.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:01 smithi187 ceph-mon[93536]: osdmap e688: 8 total, 8 up, 8 in 2024-03-29T05:05:02.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:01 smithi187 ceph-mon[93536]: failed to encode map e688 with expected crc 2024-03-29T05:05:02.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:01 smithi187 ceph-mon[93536]: failed to encode map e688 with expected crc 2024-03-29T05:05:02.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:01 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:02.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[101970]: failed to encode map e687 with expected crc 2024-03-29T05:05:02.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[101970]: failed to encode map e687 with expected crc 2024-03-29T05:05:02.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[101970]: Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-03-29T05:05:02.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31"}]': finished 2024-03-29T05:05:02.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[101970]: failed to encode map e688 with expected crc 2024-03-29T05:05:02.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[101970]: osdmap e688: 8 total, 8 up, 8 in 2024-03-29T05:05:02.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[101970]: failed to encode map e688 with expected crc 2024-03-29T05:05:02.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[101970]: failed to encode map e688 with expected crc 2024-03-29T05:05:02.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:02.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[106466]: failed to encode map e687 with expected crc 2024-03-29T05:05:02.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[106466]: failed to encode map e687 with expected crc 2024-03-29T05:05:02.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[106466]: Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-03-29T05:05:02.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi181-86707-29", "tierpool":"test-librbd-smithi181-86707-31"}]': finished 2024-03-29T05:05:02.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[106466]: failed to encode map e688 with expected crc 2024-03-29T05:05:02.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[106466]: osdmap e688: 8 total, 8 up, 8 in 2024-03-29T05:05:02.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[106466]: failed to encode map e688 with expected crc 2024-03-29T05:05:02.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[106466]: failed to encode map e688 with expected crc 2024-03-29T05:05:02.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:01 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:02.713 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:03.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:02 smithi187 ceph-mon[93536]: failed to encode map e688 with expected crc 2024-03-29T05:05:03.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:02 smithi187 ceph-mon[93536]: pgmap v738: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 1.5 KiB/s wr, 60 op/s 2024-03-29T05:05:03.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:02 smithi187 ceph-mon[93536]: osdmap e689: 8 total, 8 up, 8 in 2024-03-29T05:05:03.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:02 smithi187 ceph-mon[93536]: osdmap e690: 8 total, 8 up, 8 in 2024-03-29T05:05:03.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:02 smithi181 ceph-mon[106466]: failed to encode map e688 with expected crc 2024-03-29T05:05:03.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:02 smithi181 ceph-mon[106466]: pgmap v738: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 1.5 KiB/s wr, 60 op/s 2024-03-29T05:05:03.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:02 smithi181 ceph-mon[106466]: osdmap e689: 8 total, 8 up, 8 in 2024-03-29T05:05:03.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:02 smithi181 ceph-mon[106466]: osdmap e690: 8 total, 8 up, 8 in 2024-03-29T05:05:03.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:02 smithi181 ceph-mon[101970]: failed to encode map e688 with expected crc 2024-03-29T05:05:03.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:02 smithi181 ceph-mon[101970]: pgmap v738: 617 pgs: 617 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 1.5 KiB/s wr, 60 op/s 2024-03-29T05:05:03.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:02 smithi181 ceph-mon[101970]: osdmap e689: 8 total, 8 up, 8 in 2024-03-29T05:05:03.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:02 smithi181 ceph-mon[101970]: osdmap e690: 8 total, 8 up, 8 in 2024-03-29T05:05:04.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:03 smithi187 ceph-mon[93536]: failed to encode map e689 with expected crc 2024-03-29T05:05:04.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:03 smithi187 ceph-mon[93536]: failed to encode map e689 with expected crc 2024-03-29T05:05:04.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:03 smithi187 ceph-mon[93536]: failed to encode map e690 with expected crc 2024-03-29T05:05:04.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:03 smithi187 ceph-mon[93536]: failed to encode map e691 with expected crc 2024-03-29T05:05:04.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:03 smithi187 ceph-mon[93536]: osdmap e691: 8 total, 8 up, 8 in 2024-03-29T05:05:04.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:03 smithi181 ceph-mon[106466]: failed to encode map e689 with expected crc 2024-03-29T05:05:04.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:03 smithi181 ceph-mon[106466]: failed to encode map e689 with expected crc 2024-03-29T05:05:04.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:03 smithi181 ceph-mon[106466]: failed to encode map e690 with expected crc 2024-03-29T05:05:04.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:03 smithi181 ceph-mon[106466]: failed to encode map e691 with expected crc 2024-03-29T05:05:04.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:03 smithi181 ceph-mon[106466]: osdmap e691: 8 total, 8 up, 8 in 2024-03-29T05:05:04.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:03 smithi181 ceph-mon[101970]: failed to encode map e689 with expected crc 2024-03-29T05:05:04.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:03 smithi181 ceph-mon[101970]: failed to encode map e689 with expected crc 2024-03-29T05:05:04.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:03 smithi181 ceph-mon[101970]: failed to encode map e690 with expected crc 2024-03-29T05:05:04.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:03 smithi181 ceph-mon[101970]: failed to encode map e691 with expected crc 2024-03-29T05:05:04.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:03 smithi181 ceph-mon[101970]: osdmap e691: 8 total, 8 up, 8 in 2024-03-29T05:05:04.717 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyup (2982 ms) 2024-03-29T05:05:04.717 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyupMismatch 2024-03-29T05:05:04.717 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:05.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:04 smithi187 ceph-mon[93536]: pgmap v741: 617 pgs: 617 active+clean; 139 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 688 KiB/s wr, 969 op/s 2024-03-29T05:05:05.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:04 smithi187 ceph-mon[93536]: failed to encode map e691 with expected crc 2024-03-29T05:05:05.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:04 smithi187 ceph-mon[93536]: failed to encode map e691 with expected crc 2024-03-29T05:05:05.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:04 smithi187 ceph-mon[93536]: osdmap e692: 8 total, 8 up, 8 in 2024-03-29T05:05:05.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:04 smithi187 ceph-mon[93536]: failed to encode map e692 with expected crc 2024-03-29T05:05:05.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:04 smithi181 ceph-mon[101970]: pgmap v741: 617 pgs: 617 active+clean; 139 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 688 KiB/s wr, 969 op/s 2024-03-29T05:05:05.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:04 smithi181 ceph-mon[101970]: failed to encode map e691 with expected crc 2024-03-29T05:05:05.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:04 smithi181 ceph-mon[101970]: failed to encode map e691 with expected crc 2024-03-29T05:05:05.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:04 smithi181 ceph-mon[101970]: osdmap e692: 8 total, 8 up, 8 in 2024-03-29T05:05:05.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:04 smithi181 ceph-mon[101970]: failed to encode map e692 with expected crc 2024-03-29T05:05:05.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:04 smithi181 ceph-mon[106466]: pgmap v741: 617 pgs: 617 active+clean; 139 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 688 KiB/s wr, 969 op/s 2024-03-29T05:05:05.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:04 smithi181 ceph-mon[106466]: failed to encode map e691 with expected crc 2024-03-29T05:05:05.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:04 smithi181 ceph-mon[106466]: failed to encode map e691 with expected crc 2024-03-29T05:05:05.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:04 smithi181 ceph-mon[106466]: osdmap e692: 8 total, 8 up, 8 in 2024-03-29T05:05:05.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:04 smithi181 ceph-mon[106466]: failed to encode map e692 with expected crc 2024-03-29T05:05:05.763 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:06.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:05 smithi187 ceph-mon[93536]: failed to encode map e692 with expected crc 2024-03-29T05:05:06.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:05 smithi187 ceph-mon[93536]: failed to encode map e692 with expected crc 2024-03-29T05:05:06.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:05 smithi187 ceph-mon[93536]: failed to encode map e693 with expected crc 2024-03-29T05:05:06.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:05 smithi187 ceph-mon[93536]: osdmap e693: 8 total, 8 up, 8 in 2024-03-29T05:05:06.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:05 smithi181 ceph-mon[101970]: failed to encode map e692 with expected crc 2024-03-29T05:05:06.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:05 smithi181 ceph-mon[101970]: failed to encode map e692 with expected crc 2024-03-29T05:05:06.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:05 smithi181 ceph-mon[101970]: failed to encode map e693 with expected crc 2024-03-29T05:05:06.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:05 smithi181 ceph-mon[101970]: osdmap e693: 8 total, 8 up, 8 in 2024-03-29T05:05:06.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:05 smithi181 ceph-mon[106466]: failed to encode map e692 with expected crc 2024-03-29T05:05:06.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:05 smithi181 ceph-mon[106466]: failed to encode map e692 with expected crc 2024-03-29T05:05:06.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:05 smithi181 ceph-mon[106466]: failed to encode map e693 with expected crc 2024-03-29T05:05:06.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:05 smithi181 ceph-mon[106466]: osdmap e693: 8 total, 8 up, 8 in 2024-03-29T05:05:07.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:06 smithi187 ceph-mon[93536]: pgmap v744: 617 pgs: 617 active+clean; 139 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 688 KiB/s wr, 969 op/s 2024-03-29T05:05:07.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:06 smithi187 ceph-mon[93536]: failed to encode map e693 with expected crc 2024-03-29T05:05:07.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:06 smithi187 ceph-mon[93536]: osdmap e694: 8 total, 8 up, 8 in 2024-03-29T05:05:07.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:06 smithi187 ceph-mon[93536]: failed to encode map e694 with expected crc 2024-03-29T05:05:07.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:06 smithi181 ceph-mon[106466]: pgmap v744: 617 pgs: 617 active+clean; 139 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 688 KiB/s wr, 969 op/s 2024-03-29T05:05:07.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:06 smithi181 ceph-mon[106466]: failed to encode map e693 with expected crc 2024-03-29T05:05:07.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:06 smithi181 ceph-mon[106466]: osdmap e694: 8 total, 8 up, 8 in 2024-03-29T05:05:07.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:06 smithi181 ceph-mon[106466]: failed to encode map e694 with expected crc 2024-03-29T05:05:07.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:06 smithi181 ceph-mon[101970]: pgmap v744: 617 pgs: 617 active+clean; 139 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 688 KiB/s wr, 969 op/s 2024-03-29T05:05:07.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:06 smithi181 ceph-mon[101970]: failed to encode map e693 with expected crc 2024-03-29T05:05:07.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:06 smithi181 ceph-mon[101970]: osdmap e694: 8 total, 8 up, 8 in 2024-03-29T05:05:07.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:06 smithi181 ceph-mon[101970]: failed to encode map e694 with expected crc 2024-03-29T05:05:07.701 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyupMismatch (2984 ms) 2024-03-29T05:05:07.701 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.Flatten 2024-03-29T05:05:07.701 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:08.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:08 smithi187 ceph-mon[93536]: failed to encode map e694 with expected crc 2024-03-29T05:05:08.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:08 smithi187 ceph-mon[93536]: osdmap e695: 8 total, 8 up, 8 in 2024-03-29T05:05:08.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:08 smithi187 ceph-mon[93536]: failed to encode map e695 with expected crc 2024-03-29T05:05:08.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:08 smithi181 ceph-mon[101970]: failed to encode map e694 with expected crc 2024-03-29T05:05:08.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:08 smithi181 ceph-mon[101970]: osdmap e695: 8 total, 8 up, 8 in 2024-03-29T05:05:08.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:08 smithi181 ceph-mon[101970]: failed to encode map e695 with expected crc 2024-03-29T05:05:08.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:08 smithi181 ceph-mon[106466]: failed to encode map e694 with expected crc 2024-03-29T05:05:08.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:08 smithi181 ceph-mon[106466]: osdmap e695: 8 total, 8 up, 8 in 2024-03-29T05:05:08.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:08 smithi181 ceph-mon[106466]: failed to encode map e695 with expected crc 2024-03-29T05:05:08.762 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.Flatten (1061 ms) 2024-03-29T05:05:08.762 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.Sparsify 2024-03-29T05:05:08.763 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:08.800 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 32768 2024-03-29T05:05:08.800 INFO:tasks.workunit.client.0.smithi181.stdout:read: 32768 2024-03-29T05:05:08.800 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.Sparsify (38 ms) 2024-03-29T05:05:08.800 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.SparsifyPP 2024-03-29T05:05:08.800 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:08.837 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.SparsifyPP (37 ms) 2024-03-29T05:05:08.837 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.SnapshotLimit 2024-03-29T05:05:08.837 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:09.205 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:05:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:05:08] "GET /metrics HTTP/1.1" 200 74884 "" "Prometheus/2.43.0" 2024-03-29T05:05:09.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:09 smithi181 ceph-mon[101970]: pgmap v747: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 297 KiB/s rd, 67 KiB/s wr, 417 op/s 2024-03-29T05:05:09.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:09 smithi181 ceph-mon[101970]: failed to encode map e695 with expected crc 2024-03-29T05:05:09.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:09 smithi181 ceph-mon[101970]: osdmap e696: 8 total, 8 up, 8 in 2024-03-29T05:05:09.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:09 smithi181 ceph-mon[101970]: failed to encode map e696 with expected crc 2024-03-29T05:05:09.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:09 smithi181 ceph-mon[106466]: pgmap v747: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 297 KiB/s rd, 67 KiB/s wr, 417 op/s 2024-03-29T05:05:09.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:09 smithi181 ceph-mon[106466]: failed to encode map e695 with expected crc 2024-03-29T05:05:09.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:09 smithi181 ceph-mon[106466]: osdmap e696: 8 total, 8 up, 8 in 2024-03-29T05:05:09.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:09 smithi181 ceph-mon[106466]: failed to encode map e696 with expected crc 2024-03-29T05:05:09.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:09 smithi187 ceph-mon[93536]: pgmap v747: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 297 KiB/s rd, 67 KiB/s wr, 417 op/s 2024-03-29T05:05:09.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:09 smithi187 ceph-mon[93536]: failed to encode map e695 with expected crc 2024-03-29T05:05:09.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:09 smithi187 ceph-mon[93536]: osdmap e696: 8 total, 8 up, 8 in 2024-03-29T05:05:09.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:09 smithi187 ceph-mon[93536]: failed to encode map e696 with expected crc 2024-03-29T05:05:10.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:10 smithi187 ceph-mon[93536]: failed to encode map e696 with expected crc 2024-03-29T05:05:10.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:10 smithi187 ceph-mon[93536]: failed to encode map e696 with expected crc 2024-03-29T05:05:10.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:10 smithi187 ceph-mon[93536]: osdmap e697: 8 total, 8 up, 8 in 2024-03-29T05:05:10.467 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:10 smithi181 ceph-mon[106466]: failed to encode map e696 with expected crc 2024-03-29T05:05:10.467 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:10 smithi181 ceph-mon[106466]: failed to encode map e696 with expected crc 2024-03-29T05:05:10.467 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:10 smithi181 ceph-mon[106466]: osdmap e697: 8 total, 8 up, 8 in 2024-03-29T05:05:10.467 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:10 smithi181 ceph-mon[101970]: failed to encode map e696 with expected crc 2024-03-29T05:05:10.467 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:10 smithi181 ceph-mon[101970]: failed to encode map e696 with expected crc 2024-03-29T05:05:10.467 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:10 smithi181 ceph-mon[101970]: osdmap e697: 8 total, 8 up, 8 in 2024-03-29T05:05:11.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:11 smithi187 ceph-mon[93536]: pgmap v750: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 297 KiB/s rd, 67 KiB/s wr, 419 op/s 2024-03-29T05:05:11.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:11 smithi187 ceph-mon[93536]: failed to encode map e697 with expected crc 2024-03-29T05:05:11.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:11 smithi187 ceph-mon[93536]: failed to encode map e697 with expected crc 2024-03-29T05:05:11.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:11 smithi187 ceph-mon[93536]: osdmap e698: 8 total, 8 up, 8 in 2024-03-29T05:05:11.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:11 smithi187 ceph-mon[93536]: failed to encode map e698 with expected crc 2024-03-29T05:05:11.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:11 smithi181 ceph-mon[101970]: pgmap v750: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 297 KiB/s rd, 67 KiB/s wr, 419 op/s 2024-03-29T05:05:11.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:11 smithi181 ceph-mon[101970]: failed to encode map e697 with expected crc 2024-03-29T05:05:11.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:11 smithi181 ceph-mon[101970]: failed to encode map e697 with expected crc 2024-03-29T05:05:11.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:11 smithi181 ceph-mon[101970]: osdmap e698: 8 total, 8 up, 8 in 2024-03-29T05:05:11.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:11 smithi181 ceph-mon[101970]: failed to encode map e698 with expected crc 2024-03-29T05:05:11.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:11 smithi181 ceph-mon[106466]: pgmap v750: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 297 KiB/s rd, 67 KiB/s wr, 419 op/s 2024-03-29T05:05:11.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:11 smithi181 ceph-mon[106466]: failed to encode map e697 with expected crc 2024-03-29T05:05:11.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:11 smithi181 ceph-mon[106466]: failed to encode map e697 with expected crc 2024-03-29T05:05:11.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:11 smithi181 ceph-mon[106466]: osdmap e698: 8 total, 8 up, 8 in 2024-03-29T05:05:11.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:11 smithi181 ceph-mon[106466]: failed to encode map e698 with expected crc 2024-03-29T05:05:12.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:12 smithi187 ceph-mon[93536]: failed to encode map e698 with expected crc 2024-03-29T05:05:12.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:12 smithi187 ceph-mon[93536]: failed to encode map e698 with expected crc 2024-03-29T05:05:12.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:12.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:05:12.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:12 smithi187 ceph-mon[93536]: pgmap v753: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 1 op/s 2024-03-29T05:05:12.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:12 smithi187 ceph-mon[93536]: osdmap e699: 8 total, 8 up, 8 in 2024-03-29T05:05:12.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 2]}]: dispatch 2024-03-29T05:05:12.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]}]: dispatch 2024-03-29T05:05:12.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]}]: dispatch 2024-03-29T05:05:12.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.15", "id": [0, 2]}]: dispatch 2024-03-29T05:05:12.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.f", "id": [6, 0]}]: dispatch 2024-03-29T05:05:12.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]: dispatch 2024-03-29T05:05:12.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]: dispatch 2024-03-29T05:05:12.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.5", "id": [4, 2]}]: dispatch 2024-03-29T05:05:12.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:12 smithi187 ceph-mon[93536]: failed to encode map e699 with expected crc 2024-03-29T05:05:12.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[106466]: failed to encode map e698 with expected crc 2024-03-29T05:05:12.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[106466]: failed to encode map e698 with expected crc 2024-03-29T05:05:12.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:12.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:05:12.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[106466]: pgmap v753: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 1 op/s 2024-03-29T05:05:12.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[106466]: osdmap e699: 8 total, 8 up, 8 in 2024-03-29T05:05:12.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 2]}]: dispatch 2024-03-29T05:05:12.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]}]: dispatch 2024-03-29T05:05:12.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]}]: dispatch 2024-03-29T05:05:12.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.15", "id": [0, 2]}]: dispatch 2024-03-29T05:05:12.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.f", "id": [6, 0]}]: dispatch 2024-03-29T05:05:12.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]: dispatch 2024-03-29T05:05:12.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]: dispatch 2024-03-29T05:05:12.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.5", "id": [4, 2]}]: dispatch 2024-03-29T05:05:12.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[106466]: failed to encode map e699 with expected crc 2024-03-29T05:05:12.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[101970]: failed to encode map e698 with expected crc 2024-03-29T05:05:12.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[101970]: failed to encode map e698 with expected crc 2024-03-29T05:05:12.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:12.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:05:12.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[101970]: pgmap v753: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 1 op/s 2024-03-29T05:05:12.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[101970]: osdmap e699: 8 total, 8 up, 8 in 2024-03-29T05:05:12.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 2]}]: dispatch 2024-03-29T05:05:12.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]}]: dispatch 2024-03-29T05:05:12.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]}]: dispatch 2024-03-29T05:05:12.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.15", "id": [0, 2]}]: dispatch 2024-03-29T05:05:12.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.f", "id": [6, 0]}]: dispatch 2024-03-29T05:05:12.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]: dispatch 2024-03-29T05:05:12.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]: dispatch 2024-03-29T05:05:12.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.5", "id": [4, 2]}]: dispatch 2024-03-29T05:05:12.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:12 smithi181 ceph-mon[101970]: failed to encode map e699 with expected crc 2024-03-29T05:05:13.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:13 smithi187 ceph-mon[93536]: failed to encode map e699 with expected crc 2024-03-29T05:05:13.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:13 smithi187 ceph-mon[93536]: failed to encode map e699 with expected crc 2024-03-29T05:05:13.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:13 smithi187 ceph-mon[93536]: failed to encode map e699 with expected crc 2024-03-29T05:05:13.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:13 smithi187 ceph-mon[93536]: failed to encode map e699 with expected crc 2024-03-29T05:05:13.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 2]}]': finished 2024-03-29T05:05:13.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]}]': finished 2024-03-29T05:05:13.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]}]': finished 2024-03-29T05:05:13.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.15", "id": [0, 2]}]': finished 2024-03-29T05:05:13.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.f", "id": [6, 0]}]': finished 2024-03-29T05:05:13.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]': finished 2024-03-29T05:05:13.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]': finished 2024-03-29T05:05:13.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.5", "id": [4, 2]}]': finished 2024-03-29T05:05:13.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:13 smithi187 ceph-mon[93536]: osdmap e700: 8 total, 8 up, 8 in 2024-03-29T05:05:13.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:13 smithi187 ceph-mon[93536]: failed to encode map e700 with expected crc 2024-03-29T05:05:13.702 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.SnapshotLimit (4865 ms) 2024-03-29T05:05:13.702 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.SnapshotLimitPP 2024-03-29T05:05:13.702 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:13.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[106466]: failed to encode map e699 with expected crc 2024-03-29T05:05:13.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[106466]: failed to encode map e699 with expected crc 2024-03-29T05:05:13.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[106466]: failed to encode map e699 with expected crc 2024-03-29T05:05:13.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[106466]: failed to encode map e699 with expected crc 2024-03-29T05:05:13.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 2]}]': finished 2024-03-29T05:05:13.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]}]': finished 2024-03-29T05:05:13.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]}]': finished 2024-03-29T05:05:13.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.15", "id": [0, 2]}]': finished 2024-03-29T05:05:13.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.f", "id": [6, 0]}]': finished 2024-03-29T05:05:13.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]': finished 2024-03-29T05:05:13.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]': finished 2024-03-29T05:05:13.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.5", "id": [4, 2]}]': finished 2024-03-29T05:05:13.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[106466]: osdmap e700: 8 total, 8 up, 8 in 2024-03-29T05:05:13.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[106466]: failed to encode map e700 with expected crc 2024-03-29T05:05:13.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[101970]: failed to encode map e699 with expected crc 2024-03-29T05:05:13.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[101970]: failed to encode map e699 with expected crc 2024-03-29T05:05:13.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[101970]: failed to encode map e699 with expected crc 2024-03-29T05:05:13.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[101970]: failed to encode map e699 with expected crc 2024-03-29T05:05:13.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 2]}]': finished 2024-03-29T05:05:13.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]}]': finished 2024-03-29T05:05:13.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]}]': finished 2024-03-29T05:05:13.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.15", "id": [0, 2]}]': finished 2024-03-29T05:05:13.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.f", "id": [6, 0]}]': finished 2024-03-29T05:05:13.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]': finished 2024-03-29T05:05:13.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.6", "id": [2, 4]}]': finished 2024-03-29T05:05:13.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.5", "id": [4, 2]}]': finished 2024-03-29T05:05:13.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[101970]: osdmap e700: 8 total, 8 up, 8 in 2024-03-29T05:05:13.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:13 smithi181 ceph-mon[101970]: failed to encode map e700 with expected crc 2024-03-29T05:05:14.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:14 smithi187 ceph-mon[93536]: failed to encode map e700 with expected crc 2024-03-29T05:05:14.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:14 smithi187 ceph-mon[93536]: failed to encode map e700 with expected crc 2024-03-29T05:05:14.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:14 smithi187 ceph-mon[93536]: failed to encode map e700 with expected crc 2024-03-29T05:05:14.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:14 smithi187 ceph-mon[93536]: pgmap v756: 617 pgs: 2 peering, 615 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 157 KiB/s rd, 34 KiB/s wr, 192 op/s 2024-03-29T05:05:14.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:14 smithi187 ceph-mon[93536]: osdmap e701: 8 total, 8 up, 8 in 2024-03-29T05:05:14.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:14 smithi187 ceph-mon[93536]: failed to encode map e701 with expected crc 2024-03-29T05:05:14.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:14 smithi181 ceph-mon[101970]: failed to encode map e700 with expected crc 2024-03-29T05:05:14.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:14 smithi181 ceph-mon[101970]: failed to encode map e700 with expected crc 2024-03-29T05:05:14.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:14 smithi181 ceph-mon[101970]: failed to encode map e700 with expected crc 2024-03-29T05:05:14.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:14 smithi181 ceph-mon[101970]: pgmap v756: 617 pgs: 2 peering, 615 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 157 KiB/s rd, 34 KiB/s wr, 192 op/s 2024-03-29T05:05:14.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:14 smithi181 ceph-mon[101970]: osdmap e701: 8 total, 8 up, 8 in 2024-03-29T05:05:14.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:14 smithi181 ceph-mon[101970]: failed to encode map e701 with expected crc 2024-03-29T05:05:14.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:14 smithi181 ceph-mon[106466]: failed to encode map e700 with expected crc 2024-03-29T05:05:14.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:14 smithi181 ceph-mon[106466]: failed to encode map e700 with expected crc 2024-03-29T05:05:14.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:14 smithi181 ceph-mon[106466]: failed to encode map e700 with expected crc 2024-03-29T05:05:14.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:14 smithi181 ceph-mon[106466]: pgmap v756: 617 pgs: 2 peering, 615 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 157 KiB/s rd, 34 KiB/s wr, 192 op/s 2024-03-29T05:05:14.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:14 smithi181 ceph-mon[106466]: osdmap e701: 8 total, 8 up, 8 in 2024-03-29T05:05:14.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:14 smithi181 ceph-mon[106466]: failed to encode map e701 with expected crc 2024-03-29T05:05:15.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:15 smithi187 ceph-mon[93536]: failed to encode map e701 with expected crc 2024-03-29T05:05:15.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:15 smithi187 ceph-mon[93536]: failed to encode map e701 with expected crc 2024-03-29T05:05:15.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:15 smithi187 ceph-mon[93536]: failed to encode map e701 with expected crc 2024-03-29T05:05:15.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:15 smithi187 ceph-mon[93536]: osdmap e702: 8 total, 8 up, 8 in 2024-03-29T05:05:15.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:15 smithi187 ceph-mon[93536]: failed to encode map e702 with expected crc 2024-03-29T05:05:15.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:15 smithi181 ceph-mon[101970]: failed to encode map e701 with expected crc 2024-03-29T05:05:15.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:15 smithi181 ceph-mon[101970]: failed to encode map e701 with expected crc 2024-03-29T05:05:15.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:15 smithi181 ceph-mon[101970]: failed to encode map e701 with expected crc 2024-03-29T05:05:15.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:15 smithi181 ceph-mon[101970]: osdmap e702: 8 total, 8 up, 8 in 2024-03-29T05:05:15.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:15 smithi181 ceph-mon[101970]: failed to encode map e702 with expected crc 2024-03-29T05:05:15.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:15 smithi181 ceph-mon[106466]: failed to encode map e701 with expected crc 2024-03-29T05:05:15.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:15 smithi181 ceph-mon[106466]: failed to encode map e701 with expected crc 2024-03-29T05:05:15.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:15 smithi181 ceph-mon[106466]: failed to encode map e701 with expected crc 2024-03-29T05:05:15.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:15 smithi181 ceph-mon[106466]: osdmap e702: 8 total, 8 up, 8 in 2024-03-29T05:05:15.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:15 smithi181 ceph-mon[106466]: failed to encode map e702 with expected crc 2024-03-29T05:05:16.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:16 smithi187 ceph-mon[93536]: failed to encode map e702 with expected crc 2024-03-29T05:05:16.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:16 smithi187 ceph-mon[93536]: failed to encode map e702 with expected crc 2024-03-29T05:05:16.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:16 smithi187 ceph-mon[93536]: failed to encode map e702 with expected crc 2024-03-29T05:05:16.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:16 smithi187 ceph-mon[93536]: pgmap v759: 617 pgs: 2 peering, 615 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 157 KiB/s rd, 34 KiB/s wr, 192 op/s 2024-03-29T05:05:16.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:16 smithi187 ceph-mon[93536]: osdmap e703: 8 total, 8 up, 8 in 2024-03-29T05:05:16.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:16 smithi187 ceph-mon[93536]: failed to encode map e703 with expected crc 2024-03-29T05:05:16.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:16 smithi181 ceph-mon[101970]: failed to encode map e702 with expected crc 2024-03-29T05:05:16.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:16 smithi181 ceph-mon[101970]: failed to encode map e702 with expected crc 2024-03-29T05:05:16.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:16 smithi181 ceph-mon[101970]: failed to encode map e702 with expected crc 2024-03-29T05:05:16.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:16 smithi181 ceph-mon[101970]: pgmap v759: 617 pgs: 2 peering, 615 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 157 KiB/s rd, 34 KiB/s wr, 192 op/s 2024-03-29T05:05:16.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:16 smithi181 ceph-mon[101970]: osdmap e703: 8 total, 8 up, 8 in 2024-03-29T05:05:16.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:16 smithi181 ceph-mon[101970]: failed to encode map e703 with expected crc 2024-03-29T05:05:16.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:16 smithi181 ceph-mon[106466]: failed to encode map e702 with expected crc 2024-03-29T05:05:16.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:16 smithi181 ceph-mon[106466]: failed to encode map e702 with expected crc 2024-03-29T05:05:16.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:16 smithi181 ceph-mon[106466]: failed to encode map e702 with expected crc 2024-03-29T05:05:16.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:16 smithi181 ceph-mon[106466]: pgmap v759: 617 pgs: 2 peering, 615 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 157 KiB/s rd, 34 KiB/s wr, 192 op/s 2024-03-29T05:05:16.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:16 smithi181 ceph-mon[106466]: osdmap e703: 8 total, 8 up, 8 in 2024-03-29T05:05:16.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:16 smithi181 ceph-mon[106466]: failed to encode map e703 with expected crc 2024-03-29T05:05:17.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:17 smithi187 ceph-mon[93536]: failed to encode map e703 with expected crc 2024-03-29T05:05:17.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:17 smithi187 ceph-mon[93536]: failed to encode map e703 with expected crc 2024-03-29T05:05:17.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:17 smithi187 ceph-mon[93536]: osdmap e704: 8 total, 8 up, 8 in 2024-03-29T05:05:17.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:17 smithi187 ceph-mon[93536]: failed to encode map e704 with expected crc 2024-03-29T05:05:17.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:17 smithi181 ceph-mon[101970]: failed to encode map e703 with expected crc 2024-03-29T05:05:17.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:17 smithi181 ceph-mon[101970]: failed to encode map e703 with expected crc 2024-03-29T05:05:17.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:17 smithi181 ceph-mon[101970]: osdmap e704: 8 total, 8 up, 8 in 2024-03-29T05:05:17.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:17 smithi181 ceph-mon[101970]: failed to encode map e704 with expected crc 2024-03-29T05:05:17.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:17 smithi181 ceph-mon[106466]: failed to encode map e703 with expected crc 2024-03-29T05:05:17.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:17 smithi181 ceph-mon[106466]: failed to encode map e703 with expected crc 2024-03-29T05:05:17.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:17 smithi181 ceph-mon[106466]: osdmap e704: 8 total, 8 up, 8 in 2024-03-29T05:05:17.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:17 smithi181 ceph-mon[106466]: failed to encode map e704 with expected crc 2024-03-29T05:05:18.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:18 smithi187 ceph-mon[93536]: failed to encode map e704 with expected crc 2024-03-29T05:05:18.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:18 smithi187 ceph-mon[93536]: failed to encode map e704 with expected crc 2024-03-29T05:05:18.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:18 smithi187 ceph-mon[93536]: pgmap v762: 617 pgs: 2 peering, 615 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 3.2 KiB/s wr, 34 op/s; 0 B/s, 58 keys/s, 0 objects/s recovering 2024-03-29T05:05:18.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:18 smithi187 ceph-mon[93536]: osdmap e705: 8 total, 8 up, 8 in 2024-03-29T05:05:18.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:18 smithi187 ceph-mon[93536]: failed to encode map e705 with expected crc 2024-03-29T05:05:18.698 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.SnapshotLimitPP (4995 ms) 2024-03-29T05:05:18.698 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.RebuildObjectMapViaLockOwner 2024-03-29T05:05:18.698 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:18.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:18 smithi181 ceph-mon[101970]: failed to encode map e704 with expected crc 2024-03-29T05:05:18.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:18 smithi181 ceph-mon[101970]: failed to encode map e704 with expected crc 2024-03-29T05:05:18.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:18 smithi181 ceph-mon[101970]: pgmap v762: 617 pgs: 2 peering, 615 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 3.2 KiB/s wr, 34 op/s; 0 B/s, 58 keys/s, 0 objects/s recovering 2024-03-29T05:05:18.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:18 smithi181 ceph-mon[101970]: osdmap e705: 8 total, 8 up, 8 in 2024-03-29T05:05:18.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:18 smithi181 ceph-mon[101970]: failed to encode map e705 with expected crc 2024-03-29T05:05:18.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:18 smithi181 ceph-mon[106466]: failed to encode map e704 with expected crc 2024-03-29T05:05:18.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:18 smithi181 ceph-mon[106466]: failed to encode map e704 with expected crc 2024-03-29T05:05:18.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:18 smithi181 ceph-mon[106466]: pgmap v762: 617 pgs: 2 peering, 615 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 3.2 KiB/s wr, 34 op/s; 0 B/s, 58 keys/s, 0 objects/s recovering 2024-03-29T05:05:18.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:18 smithi181 ceph-mon[106466]: osdmap e705: 8 total, 8 up, 8 in 2024-03-29T05:05:18.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:18 smithi181 ceph-mon[106466]: failed to encode map e705 with expected crc 2024-03-29T05:05:18.754 INFO:tasks.workunit.client.0.smithi181.stdout:0.00% done 2024-03-29T05:05:18.754 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.RebuildObjectMapViaLockOwner (57 ms) 2024-03-29T05:05:18.754 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.RenameViaLockOwner 2024-03-29T05:05:18.754 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:18.818 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.RenameViaLockOwner (64 ms) 2024-03-29T05:05:18.819 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.SnapCreateViaLockOwner 2024-03-29T05:05:18.819 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:19.205 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:05:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:05:18] "GET /metrics HTTP/1.1" 200 74970 "" "Prometheus/2.43.0" 2024-03-29T05:05:19.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:19 smithi187 ceph-mon[93536]: failed to encode map e705 with expected crc 2024-03-29T05:05:19.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:19 smithi187 ceph-mon[93536]: osdmap e706: 8 total, 8 up, 8 in 2024-03-29T05:05:19.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:19 smithi187 ceph-mon[93536]: failed to encode map e706 with expected crc 2024-03-29T05:05:19.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:19 smithi181 ceph-mon[101970]: failed to encode map e705 with expected crc 2024-03-29T05:05:19.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:19 smithi181 ceph-mon[101970]: osdmap e706: 8 total, 8 up, 8 in 2024-03-29T05:05:19.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:19 smithi181 ceph-mon[101970]: failed to encode map e706 with expected crc 2024-03-29T05:05:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:19 smithi181 ceph-mon[106466]: failed to encode map e705 with expected crc 2024-03-29T05:05:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:19 smithi181 ceph-mon[106466]: osdmap e706: 8 total, 8 up, 8 in 2024-03-29T05:05:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:19 smithi181 ceph-mon[106466]: failed to encode map e706 with expected crc 2024-03-29T05:05:19.712 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.SnapCreateViaLockOwner (894 ms) 2024-03-29T05:05:19.712 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.SnapRemoveViaLockOwner 2024-03-29T05:05:19.712 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:20.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:20 smithi187 ceph-mon[93536]: failed to encode map e706 with expected crc 2024-03-29T05:05:20.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:20 smithi187 ceph-mon[93536]: failed to encode map e706 with expected crc 2024-03-29T05:05:20.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:20 smithi187 ceph-mon[93536]: failed to encode map e706 with expected crc 2024-03-29T05:05:20.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:20 smithi187 ceph-mon[93536]: pgmap v765: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.5 KiB/s wr, 37 op/s; 0 B/s, 58 keys/s, 0 objects/s recovering 2024-03-29T05:05:20.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:20 smithi187 ceph-mon[93536]: osdmap e707: 8 total, 8 up, 8 in 2024-03-29T05:05:20.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:20 smithi187 ceph-mon[93536]: failed to encode map e707 with expected crc 2024-03-29T05:05:20.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:20 smithi181 ceph-mon[101970]: failed to encode map e706 with expected crc 2024-03-29T05:05:20.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:20 smithi181 ceph-mon[101970]: failed to encode map e706 with expected crc 2024-03-29T05:05:20.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:20 smithi181 ceph-mon[101970]: failed to encode map e706 with expected crc 2024-03-29T05:05:20.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:20 smithi181 ceph-mon[101970]: pgmap v765: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.5 KiB/s wr, 37 op/s; 0 B/s, 58 keys/s, 0 objects/s recovering 2024-03-29T05:05:20.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:20 smithi181 ceph-mon[101970]: osdmap e707: 8 total, 8 up, 8 in 2024-03-29T05:05:20.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:20 smithi181 ceph-mon[101970]: failed to encode map e707 with expected crc 2024-03-29T05:05:20.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:20 smithi181 ceph-mon[106466]: failed to encode map e706 with expected crc 2024-03-29T05:05:20.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:20 smithi181 ceph-mon[106466]: failed to encode map e706 with expected crc 2024-03-29T05:05:20.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:20 smithi181 ceph-mon[106466]: failed to encode map e706 with expected crc 2024-03-29T05:05:20.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:20 smithi181 ceph-mon[106466]: pgmap v765: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.5 KiB/s wr, 37 op/s; 0 B/s, 58 keys/s, 0 objects/s recovering 2024-03-29T05:05:20.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:20 smithi181 ceph-mon[106466]: osdmap e707: 8 total, 8 up, 8 in 2024-03-29T05:05:20.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:20 smithi181 ceph-mon[106466]: failed to encode map e707 with expected crc 2024-03-29T05:05:21.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:21 smithi187 ceph-mon[93536]: failed to encode map e707 with expected crc 2024-03-29T05:05:21.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:21 smithi187 ceph-mon[93536]: failed to encode map e707 with expected crc 2024-03-29T05:05:21.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:21 smithi187 ceph-mon[93536]: osdmap e708: 8 total, 8 up, 8 in 2024-03-29T05:05:21.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:21 smithi187 ceph-mon[93536]: failed to encode map e708 with expected crc 2024-03-29T05:05:21.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:21 smithi181 ceph-mon[101970]: failed to encode map e707 with expected crc 2024-03-29T05:05:21.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:21 smithi181 ceph-mon[101970]: failed to encode map e707 with expected crc 2024-03-29T05:05:21.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:21 smithi181 ceph-mon[101970]: osdmap e708: 8 total, 8 up, 8 in 2024-03-29T05:05:21.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:21 smithi181 ceph-mon[101970]: failed to encode map e708 with expected crc 2024-03-29T05:05:21.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:21 smithi181 ceph-mon[106466]: failed to encode map e707 with expected crc 2024-03-29T05:05:21.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:21 smithi181 ceph-mon[106466]: failed to encode map e707 with expected crc 2024-03-29T05:05:21.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:21 smithi181 ceph-mon[106466]: osdmap e708: 8 total, 8 up, 8 in 2024-03-29T05:05:21.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:21 smithi181 ceph-mon[106466]: failed to encode map e708 with expected crc 2024-03-29T05:05:21.720 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.SnapRemoveViaLockOwner (2008 ms) 2024-03-29T05:05:21.721 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.UpdateFeaturesViaLockOwner 2024-03-29T05:05:21.721 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:21.778 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.UpdateFeaturesViaLockOwner (57 ms) 2024-03-29T05:05:21.778 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.EnableJournalingViaLockOwner 2024-03-29T05:05:21.778 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T05:05:21.778 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.EnableJournalingViaLockOwner (0 ms) 2024-03-29T05:05:21.778 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.SnapRemove2 2024-03-29T05:05:21.778 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:22.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:22 smithi187 ceph-mon[93536]: failed to encode map e708 with expected crc 2024-03-29T05:05:22.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:22 smithi187 ceph-mon[93536]: failed to encode map e708 with expected crc 2024-03-29T05:05:22.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:22 smithi187 ceph-mon[93536]: pgmap v768: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-29T05:05:22.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:22 smithi187 ceph-mon[93536]: failed to encode map e709 with expected crc 2024-03-29T05:05:22.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:22 smithi187 ceph-mon[93536]: osdmap e709: 8 total, 8 up, 8 in 2024-03-29T05:05:22.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:22 smithi181 ceph-mon[101970]: failed to encode map e708 with expected crc 2024-03-29T05:05:22.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:22 smithi181 ceph-mon[101970]: failed to encode map e708 with expected crc 2024-03-29T05:05:22.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:22 smithi181 ceph-mon[101970]: pgmap v768: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-29T05:05:22.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:22 smithi181 ceph-mon[101970]: failed to encode map e709 with expected crc 2024-03-29T05:05:22.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:22 smithi181 ceph-mon[101970]: osdmap e709: 8 total, 8 up, 8 in 2024-03-29T05:05:22.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:22 smithi181 ceph-mon[106466]: failed to encode map e708 with expected crc 2024-03-29T05:05:22.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:22 smithi181 ceph-mon[106466]: failed to encode map e708 with expected crc 2024-03-29T05:05:22.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:22 smithi181 ceph-mon[106466]: pgmap v768: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-29T05:05:22.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:22 smithi181 ceph-mon[106466]: failed to encode map e709 with expected crc 2024-03-29T05:05:22.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:22 smithi181 ceph-mon[106466]: osdmap e709: 8 total, 8 up, 8 in 2024-03-29T05:05:23.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:23 smithi187 ceph-mon[93536]: failed to encode map e709 with expected crc 2024-03-29T05:05:23.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:23 smithi187 ceph-mon[93536]: failed to encode map e709 with expected crc 2024-03-29T05:05:23.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:23 smithi187 ceph-mon[93536]: osdmap e710: 8 total, 8 up, 8 in 2024-03-29T05:05:23.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:23 smithi187 ceph-mon[93536]: failed to encode map e710 with expected crc 2024-03-29T05:05:23.695 INFO:tasks.workunit.client.0.smithi181.stdout:100.00% done 2024-03-29T05:05:23.695 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.SnapRemove2 (1918 ms) 2024-03-29T05:05:23.696 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.SnapRenameViaLockOwner 2024-03-29T05:05:23.696 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:23.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:23 smithi181 ceph-mon[101970]: failed to encode map e709 with expected crc 2024-03-29T05:05:23.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:23 smithi181 ceph-mon[101970]: failed to encode map e709 with expected crc 2024-03-29T05:05:23.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:23 smithi181 ceph-mon[101970]: osdmap e710: 8 total, 8 up, 8 in 2024-03-29T05:05:23.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:23 smithi181 ceph-mon[101970]: failed to encode map e710 with expected crc 2024-03-29T05:05:23.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:23 smithi181 ceph-mon[106466]: failed to encode map e709 with expected crc 2024-03-29T05:05:23.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:23 smithi181 ceph-mon[106466]: failed to encode map e709 with expected crc 2024-03-29T05:05:23.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:23 smithi181 ceph-mon[106466]: osdmap e710: 8 total, 8 up, 8 in 2024-03-29T05:05:23.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:23 smithi181 ceph-mon[106466]: failed to encode map e710 with expected crc 2024-03-29T05:05:24.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:24 smithi187 ceph-mon[93536]: failed to encode map e710 with expected crc 2024-03-29T05:05:24.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:24 smithi187 ceph-mon[93536]: failed to encode map e710 with expected crc 2024-03-29T05:05:24.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:24 smithi187 ceph-mon[93536]: failed to encode map e710 with expected crc 2024-03-29T05:05:24.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:24 smithi187 ceph-mon[93536]: pgmap v771: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 130 KiB/s rd, 13 KiB/s wr, 185 op/s 2024-03-29T05:05:24.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:24 smithi187 ceph-mon[93536]: osdmap e711: 8 total, 8 up, 8 in 2024-03-29T05:05:24.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:24 smithi187 ceph-mon[93536]: failed to encode map e711 with expected crc 2024-03-29T05:05:24.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:24 smithi181 ceph-mon[106466]: failed to encode map e710 with expected crc 2024-03-29T05:05:24.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:24 smithi181 ceph-mon[106466]: failed to encode map e710 with expected crc 2024-03-29T05:05:24.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:24 smithi181 ceph-mon[106466]: failed to encode map e710 with expected crc 2024-03-29T05:05:24.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:24 smithi181 ceph-mon[106466]: pgmap v771: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 130 KiB/s rd, 13 KiB/s wr, 185 op/s 2024-03-29T05:05:24.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:24 smithi181 ceph-mon[106466]: osdmap e711: 8 total, 8 up, 8 in 2024-03-29T05:05:24.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:24 smithi181 ceph-mon[106466]: failed to encode map e711 with expected crc 2024-03-29T05:05:24.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:24 smithi181 ceph-mon[101970]: failed to encode map e710 with expected crc 2024-03-29T05:05:24.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:24 smithi181 ceph-mon[101970]: failed to encode map e710 with expected crc 2024-03-29T05:05:24.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:24 smithi181 ceph-mon[101970]: failed to encode map e710 with expected crc 2024-03-29T05:05:24.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:24 smithi181 ceph-mon[101970]: pgmap v771: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 130 KiB/s rd, 13 KiB/s wr, 185 op/s 2024-03-29T05:05:24.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:24 smithi181 ceph-mon[101970]: osdmap e711: 8 total, 8 up, 8 in 2024-03-29T05:05:24.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:24 smithi181 ceph-mon[101970]: failed to encode map e711 with expected crc 2024-03-29T05:05:24.713 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.SnapRenameViaLockOwner (1017 ms) 2024-03-29T05:05:24.713 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.SnapProtectViaLockOwner 2024-03-29T05:05:24.713 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:25.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:25 smithi187 ceph-mon[93536]: failed to encode map e711 with expected crc 2024-03-29T05:05:25.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:25 smithi187 ceph-mon[93536]: failed to encode map e711 with expected crc 2024-03-29T05:05:25.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:25 smithi187 ceph-mon[93536]: osdmap e712: 8 total, 8 up, 8 in 2024-03-29T05:05:25.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:25 smithi181 ceph-mon[101970]: failed to encode map e711 with expected crc 2024-03-29T05:05:25.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:25 smithi181 ceph-mon[101970]: failed to encode map e711 with expected crc 2024-03-29T05:05:25.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:25 smithi181 ceph-mon[101970]: osdmap e712: 8 total, 8 up, 8 in 2024-03-29T05:05:25.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:25 smithi181 ceph-mon[106466]: failed to encode map e711 with expected crc 2024-03-29T05:05:25.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:25 smithi181 ceph-mon[106466]: failed to encode map e711 with expected crc 2024-03-29T05:05:25.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:25 smithi181 ceph-mon[106466]: osdmap e712: 8 total, 8 up, 8 in 2024-03-29T05:05:25.719 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.SnapProtectViaLockOwner (1007 ms) 2024-03-29T05:05:25.719 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.SnapUnprotectViaLockOwner 2024-03-29T05:05:25.719 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:26.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:26 smithi187 ceph-mon[93536]: failed to encode map e712 with expected crc 2024-03-29T05:05:26.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:26 smithi187 ceph-mon[93536]: failed to encode map e712 with expected crc 2024-03-29T05:05:26.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:26 smithi187 ceph-mon[93536]: pgmap v774: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 130 KiB/s rd, 13 KiB/s wr, 185 op/s 2024-03-29T05:05:26.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:26 smithi187 ceph-mon[93536]: osdmap e713: 8 total, 8 up, 8 in 2024-03-29T05:05:26.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:05:26.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:26 smithi181 ceph-mon[101970]: failed to encode map e712 with expected crc 2024-03-29T05:05:26.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:26 smithi181 ceph-mon[101970]: failed to encode map e712 with expected crc 2024-03-29T05:05:26.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:26 smithi181 ceph-mon[101970]: pgmap v774: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 130 KiB/s rd, 13 KiB/s wr, 185 op/s 2024-03-29T05:05:26.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:26 smithi181 ceph-mon[101970]: osdmap e713: 8 total, 8 up, 8 in 2024-03-29T05:05:26.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:05:26.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:26 smithi181 ceph-mon[106466]: failed to encode map e712 with expected crc 2024-03-29T05:05:26.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:26 smithi181 ceph-mon[106466]: failed to encode map e712 with expected crc 2024-03-29T05:05:26.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:26 smithi181 ceph-mon[106466]: pgmap v774: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 130 KiB/s rd, 13 KiB/s wr, 185 op/s 2024-03-29T05:05:26.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:26 smithi181 ceph-mon[106466]: osdmap e713: 8 total, 8 up, 8 in 2024-03-29T05:05:26.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:05:26.723 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.SnapUnprotectViaLockOwner (1004 ms) 2024-03-29T05:05:26.723 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.FlattenViaLockOwner 2024-03-29T05:05:26.723 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:27.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:27 smithi187 ceph-mon[93536]: failed to encode map e713 with expected crc 2024-03-29T05:05:27.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:27 smithi187 ceph-mon[93536]: failed to encode map e713 with expected crc 2024-03-29T05:05:27.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:27 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:27.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:27 smithi187 ceph-mon[93536]: osdmap e714: 8 total, 8 up, 8 in 2024-03-29T05:05:27.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:27 smithi187 ceph-mon[93536]: failed to encode map e714 with expected crc 2024-03-29T05:05:27.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:27 smithi181 ceph-mon[106466]: failed to encode map e713 with expected crc 2024-03-29T05:05:27.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:27 smithi181 ceph-mon[106466]: failed to encode map e713 with expected crc 2024-03-29T05:05:27.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:27 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:27.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:27 smithi181 ceph-mon[106466]: osdmap e714: 8 total, 8 up, 8 in 2024-03-29T05:05:27.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:27 smithi181 ceph-mon[106466]: failed to encode map e714 with expected crc 2024-03-29T05:05:27.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:27 smithi181 ceph-mon[101970]: failed to encode map e713 with expected crc 2024-03-29T05:05:27.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:27 smithi181 ceph-mon[101970]: failed to encode map e713 with expected crc 2024-03-29T05:05:27.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:27 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:27.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:27 smithi181 ceph-mon[101970]: osdmap e714: 8 total, 8 up, 8 in 2024-03-29T05:05:27.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:27 smithi181 ceph-mon[101970]: failed to encode map e714 with expected crc 2024-03-29T05:05:27.768 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.FlattenViaLockOwner (1045 ms) 2024-03-29T05:05:27.768 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ResizeViaLockOwner 2024-03-29T05:05:27.768 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:27.813 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ResizeViaLockOwner (45 ms) 2024-03-29T05:05:27.813 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.SparsifyViaLockOwner 2024-03-29T05:05:27.813 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:27.848 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.SparsifyViaLockOwner (35 ms) 2024-03-29T05:05:27.848 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ObjectMapConsistentSnap 2024-03-29T05:05:27.848 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:28.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:28 smithi187 ceph-mon[93536]: failed to encode map e714 with expected crc 2024-03-29T05:05:28.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:28 smithi187 ceph-mon[93536]: failed to encode map e714 with expected crc 2024-03-29T05:05:28.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:28 smithi187 ceph-mon[93536]: pgmap v777: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 157 KiB/s rd, 10 KiB/s wr, 205 op/s 2024-03-29T05:05:28.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:28 smithi187 ceph-mon[93536]: osdmap e715: 8 total, 8 up, 8 in 2024-03-29T05:05:28.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:28 smithi187 ceph-mon[93536]: failed to encode map e715 with expected crc 2024-03-29T05:05:28.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:28 smithi181 ceph-mon[106466]: failed to encode map e714 with expected crc 2024-03-29T05:05:28.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:28 smithi181 ceph-mon[106466]: failed to encode map e714 with expected crc 2024-03-29T05:05:28.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:28 smithi181 ceph-mon[106466]: pgmap v777: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 157 KiB/s rd, 10 KiB/s wr, 205 op/s 2024-03-29T05:05:28.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:28 smithi181 ceph-mon[106466]: osdmap e715: 8 total, 8 up, 8 in 2024-03-29T05:05:28.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:28 smithi181 ceph-mon[106466]: failed to encode map e715 with expected crc 2024-03-29T05:05:28.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:28 smithi181 ceph-mon[101970]: failed to encode map e714 with expected crc 2024-03-29T05:05:28.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:28 smithi181 ceph-mon[101970]: failed to encode map e714 with expected crc 2024-03-29T05:05:28.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:28 smithi181 ceph-mon[101970]: pgmap v777: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 157 KiB/s rd, 10 KiB/s wr, 205 op/s 2024-03-29T05:05:28.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:28 smithi181 ceph-mon[101970]: osdmap e715: 8 total, 8 up, 8 in 2024-03-29T05:05:28.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:28 smithi181 ceph-mon[101970]: failed to encode map e715 with expected crc 2024-03-29T05:05:29.206 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:05:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:05:28] "GET /metrics HTTP/1.1" 200 74969 "" "Prometheus/2.43.0" 2024-03-29T05:05:29.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:29 smithi187 ceph-mon[93536]: failed to encode map e715 with expected crc 2024-03-29T05:05:29.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:29 smithi187 ceph-mon[93536]: failed to encode map e715 with expected crc 2024-03-29T05:05:29.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:29 smithi187 ceph-mon[93536]: osdmap e716: 8 total, 8 up, 8 in 2024-03-29T05:05:29.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:29 smithi181 ceph-mon[106466]: failed to encode map e715 with expected crc 2024-03-29T05:05:29.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:29 smithi181 ceph-mon[106466]: failed to encode map e715 with expected crc 2024-03-29T05:05:29.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:29 smithi181 ceph-mon[106466]: osdmap e716: 8 total, 8 up, 8 in 2024-03-29T05:05:29.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:29 smithi181 ceph-mon[101970]: failed to encode map e715 with expected crc 2024-03-29T05:05:29.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:29 smithi181 ceph-mon[101970]: failed to encode map e715 with expected crc 2024-03-29T05:05:29.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:29 smithi181 ceph-mon[101970]: osdmap e716: 8 total, 8 up, 8 in 2024-03-29T05:05:30.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:30 smithi187 ceph-mon[93536]: failed to encode map e716 with expected crc 2024-03-29T05:05:30.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:30 smithi187 ceph-mon[93536]: failed to encode map e716 with expected crc 2024-03-29T05:05:30.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:30 smithi187 ceph-mon[93536]: pgmap v780: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 194 KiB/s rd, 13 KiB/s wr, 256 op/s 2024-03-29T05:05:30.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:30 smithi187 ceph-mon[93536]: osdmap e717: 8 total, 8 up, 8 in 2024-03-29T05:05:30.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:30 smithi187 ceph-mon[93536]: failed to encode map e717 with expected crc 2024-03-29T05:05:30.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:30 smithi181 ceph-mon[106466]: failed to encode map e716 with expected crc 2024-03-29T05:05:30.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:30 smithi181 ceph-mon[106466]: failed to encode map e716 with expected crc 2024-03-29T05:05:30.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:30 smithi181 ceph-mon[106466]: pgmap v780: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 194 KiB/s rd, 13 KiB/s wr, 256 op/s 2024-03-29T05:05:30.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:30 smithi181 ceph-mon[106466]: osdmap e717: 8 total, 8 up, 8 in 2024-03-29T05:05:30.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:30 smithi181 ceph-mon[106466]: failed to encode map e717 with expected crc 2024-03-29T05:05:30.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:30 smithi181 ceph-mon[101970]: failed to encode map e716 with expected crc 2024-03-29T05:05:30.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:30 smithi181 ceph-mon[101970]: failed to encode map e716 with expected crc 2024-03-29T05:05:30.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:30 smithi181 ceph-mon[101970]: pgmap v780: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 194 KiB/s rd, 13 KiB/s wr, 256 op/s 2024-03-29T05:05:30.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:30 smithi181 ceph-mon[101970]: osdmap e717: 8 total, 8 up, 8 in 2024-03-29T05:05:30.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:30 smithi181 ceph-mon[101970]: failed to encode map e717 with expected crc 2024-03-29T05:05:31.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:31 smithi187 ceph-mon[93536]: failed to encode map e717 with expected crc 2024-03-29T05:05:31.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:31 smithi187 ceph-mon[93536]: failed to encode map e717 with expected crc 2024-03-29T05:05:31.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:31 smithi187 ceph-mon[93536]: osdmap e718: 8 total, 8 up, 8 in 2024-03-29T05:05:31.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:31 smithi187 ceph-mon[93536]: failed to encode map e718 with expected crc 2024-03-29T05:05:31.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:31 smithi181 ceph-mon[101970]: failed to encode map e717 with expected crc 2024-03-29T05:05:31.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:31 smithi181 ceph-mon[101970]: failed to encode map e717 with expected crc 2024-03-29T05:05:31.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:31 smithi181 ceph-mon[101970]: osdmap e718: 8 total, 8 up, 8 in 2024-03-29T05:05:31.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:31 smithi181 ceph-mon[101970]: failed to encode map e718 with expected crc 2024-03-29T05:05:31.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:31 smithi181 ceph-mon[106466]: failed to encode map e717 with expected crc 2024-03-29T05:05:31.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:31 smithi181 ceph-mon[106466]: failed to encode map e717 with expected crc 2024-03-29T05:05:31.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:31 smithi181 ceph-mon[106466]: osdmap e718: 8 total, 8 up, 8 in 2024-03-29T05:05:31.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:31 smithi181 ceph-mon[106466]: failed to encode map e718 with expected crc 2024-03-29T05:05:32.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:32 smithi187 ceph-mon[93536]: failed to encode map e718 with expected crc 2024-03-29T05:05:32.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:32 smithi187 ceph-mon[93536]: failed to encode map e718 with expected crc 2024-03-29T05:05:32.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:32 smithi187 ceph-mon[93536]: pgmap v783: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 2.5 KiB/s wr, 50 op/s 2024-03-29T05:05:32.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:32 smithi187 ceph-mon[93536]: osdmap e719: 8 total, 8 up, 8 in 2024-03-29T05:05:32.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:32 smithi181 ceph-mon[101970]: failed to encode map e718 with expected crc 2024-03-29T05:05:32.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:32 smithi181 ceph-mon[101970]: failed to encode map e718 with expected crc 2024-03-29T05:05:32.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:32 smithi181 ceph-mon[101970]: pgmap v783: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 2.5 KiB/s wr, 50 op/s 2024-03-29T05:05:32.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:32 smithi181 ceph-mon[101970]: osdmap e719: 8 total, 8 up, 8 in 2024-03-29T05:05:32.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:32 smithi181 ceph-mon[106466]: failed to encode map e718 with expected crc 2024-03-29T05:05:32.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:32 smithi181 ceph-mon[106466]: failed to encode map e718 with expected crc 2024-03-29T05:05:32.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:32 smithi181 ceph-mon[106466]: pgmap v783: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 2.5 KiB/s wr, 50 op/s 2024-03-29T05:05:32.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:32 smithi181 ceph-mon[106466]: osdmap e719: 8 total, 8 up, 8 in 2024-03-29T05:05:33.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:33 smithi187 ceph-mon[93536]: failed to encode map e719 with expected crc 2024-03-29T05:05:33.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:33 smithi187 ceph-mon[93536]: failed to encode map e719 with expected crc 2024-03-29T05:05:33.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:33 smithi187 ceph-mon[93536]: failed to encode map e719 with expected crc 2024-03-29T05:05:33.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:33 smithi187 ceph-mon[93536]: failed to encode map e720 with expected crc 2024-03-29T05:05:33.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:33 smithi187 ceph-mon[93536]: osdmap e720: 8 total, 8 up, 8 in 2024-03-29T05:05:33.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:33 smithi181 ceph-mon[101970]: failed to encode map e719 with expected crc 2024-03-29T05:05:33.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:33 smithi181 ceph-mon[101970]: failed to encode map e719 with expected crc 2024-03-29T05:05:33.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:33 smithi181 ceph-mon[101970]: failed to encode map e719 with expected crc 2024-03-29T05:05:33.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:33 smithi181 ceph-mon[101970]: failed to encode map e720 with expected crc 2024-03-29T05:05:33.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:33 smithi181 ceph-mon[101970]: osdmap e720: 8 total, 8 up, 8 in 2024-03-29T05:05:33.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:33 smithi181 ceph-mon[106466]: failed to encode map e719 with expected crc 2024-03-29T05:05:33.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:33 smithi181 ceph-mon[106466]: failed to encode map e719 with expected crc 2024-03-29T05:05:33.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:33 smithi181 ceph-mon[106466]: failed to encode map e719 with expected crc 2024-03-29T05:05:33.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:33 smithi181 ceph-mon[106466]: failed to encode map e720 with expected crc 2024-03-29T05:05:33.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:33 smithi181 ceph-mon[106466]: osdmap e720: 8 total, 8 up, 8 in 2024-03-29T05:05:34.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:34 smithi187 ceph-mon[93536]: failed to encode map e720 with expected crc 2024-03-29T05:05:34.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:34 smithi187 ceph-mon[93536]: failed to encode map e720 with expected crc 2024-03-29T05:05:34.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:34 smithi187 ceph-mon[93536]: pgmap v786: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 139 KiB/s rd, 8.2 KiB/s wr, 183 op/s 2024-03-29T05:05:34.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:34 smithi187 ceph-mon[93536]: osdmap e721: 8 total, 8 up, 8 in 2024-03-29T05:05:34.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:34 smithi187 ceph-mon[93536]: failed to encode map e721 with expected crc 2024-03-29T05:05:34.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:34 smithi181 ceph-mon[106466]: failed to encode map e720 with expected crc 2024-03-29T05:05:34.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:34 smithi181 ceph-mon[106466]: failed to encode map e720 with expected crc 2024-03-29T05:05:34.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:34 smithi181 ceph-mon[106466]: pgmap v786: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 139 KiB/s rd, 8.2 KiB/s wr, 183 op/s 2024-03-29T05:05:34.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:34 smithi181 ceph-mon[106466]: osdmap e721: 8 total, 8 up, 8 in 2024-03-29T05:05:34.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:34 smithi181 ceph-mon[106466]: failed to encode map e721 with expected crc 2024-03-29T05:05:34.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:34 smithi181 ceph-mon[101970]: failed to encode map e720 with expected crc 2024-03-29T05:05:34.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:34 smithi181 ceph-mon[101970]: failed to encode map e720 with expected crc 2024-03-29T05:05:34.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:34 smithi181 ceph-mon[101970]: pgmap v786: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 139 KiB/s rd, 8.2 KiB/s wr, 183 op/s 2024-03-29T05:05:34.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:34 smithi181 ceph-mon[101970]: osdmap e721: 8 total, 8 up, 8 in 2024-03-29T05:05:34.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:34 smithi181 ceph-mon[101970]: failed to encode map e721 with expected crc 2024-03-29T05:05:35.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:35 smithi187 ceph-mon[93536]: failed to encode map e721 with expected crc 2024-03-29T05:05:35.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:35 smithi187 ceph-mon[93536]: failed to encode map e721 with expected crc 2024-03-29T05:05:35.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:35 smithi187 ceph-mon[93536]: osdmap e722: 8 total, 8 up, 8 in 2024-03-29T05:05:35.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:35 smithi187 ceph-mon[93536]: failed to encode map e722 with expected crc 2024-03-29T05:05:35.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:35 smithi181 ceph-mon[101970]: failed to encode map e721 with expected crc 2024-03-29T05:05:35.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:35 smithi181 ceph-mon[101970]: failed to encode map e721 with expected crc 2024-03-29T05:05:35.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:35 smithi181 ceph-mon[101970]: osdmap e722: 8 total, 8 up, 8 in 2024-03-29T05:05:35.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:35 smithi181 ceph-mon[101970]: failed to encode map e722 with expected crc 2024-03-29T05:05:35.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:35 smithi181 ceph-mon[106466]: failed to encode map e721 with expected crc 2024-03-29T05:05:35.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:35 smithi181 ceph-mon[106466]: failed to encode map e721 with expected crc 2024-03-29T05:05:35.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:35 smithi181 ceph-mon[106466]: osdmap e722: 8 total, 8 up, 8 in 2024-03-29T05:05:35.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:35 smithi181 ceph-mon[106466]: failed to encode map e722 with expected crc 2024-03-29T05:05:36.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:36 smithi187 ceph-mon[93536]: failed to encode map e722 with expected crc 2024-03-29T05:05:36.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:36 smithi187 ceph-mon[93536]: failed to encode map e722 with expected crc 2024-03-29T05:05:36.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:36 smithi187 ceph-mon[93536]: failed to encode map e722 with expected crc 2024-03-29T05:05:36.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:36 smithi187 ceph-mon[93536]: pgmap v789: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 139 KiB/s rd, 8.2 KiB/s wr, 183 op/s 2024-03-29T05:05:36.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:36 smithi187 ceph-mon[93536]: osdmap e723: 8 total, 8 up, 8 in 2024-03-29T05:05:36.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:36 smithi187 ceph-mon[93536]: failed to encode map e723 with expected crc 2024-03-29T05:05:36.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:36 smithi181 ceph-mon[101970]: failed to encode map e722 with expected crc 2024-03-29T05:05:36.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:36 smithi181 ceph-mon[101970]: failed to encode map e722 with expected crc 2024-03-29T05:05:36.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:36 smithi181 ceph-mon[101970]: failed to encode map e722 with expected crc 2024-03-29T05:05:36.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:36 smithi181 ceph-mon[101970]: pgmap v789: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 139 KiB/s rd, 8.2 KiB/s wr, 183 op/s 2024-03-29T05:05:36.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:36 smithi181 ceph-mon[101970]: osdmap e723: 8 total, 8 up, 8 in 2024-03-29T05:05:36.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:36 smithi181 ceph-mon[101970]: failed to encode map e723 with expected crc 2024-03-29T05:05:36.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:36 smithi181 ceph-mon[106466]: failed to encode map e722 with expected crc 2024-03-29T05:05:36.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:36 smithi181 ceph-mon[106466]: failed to encode map e722 with expected crc 2024-03-29T05:05:36.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:36 smithi181 ceph-mon[106466]: failed to encode map e722 with expected crc 2024-03-29T05:05:36.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:36 smithi181 ceph-mon[106466]: pgmap v789: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 139 KiB/s rd, 8.2 KiB/s wr, 183 op/s 2024-03-29T05:05:36.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:36 smithi181 ceph-mon[106466]: osdmap e723: 8 total, 8 up, 8 in 2024-03-29T05:05:36.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:36 smithi181 ceph-mon[106466]: failed to encode map e723 with expected crc 2024-03-29T05:05:37.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:37 smithi187 ceph-mon[93536]: failed to encode map e723 with expected crc 2024-03-29T05:05:37.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:37 smithi187 ceph-mon[93536]: failed to encode map e723 with expected crc 2024-03-29T05:05:37.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:37 smithi187 ceph-mon[93536]: osdmap e724: 8 total, 8 up, 8 in 2024-03-29T05:05:37.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:37 smithi187 ceph-mon[93536]: failed to encode map e724 with expected crc 2024-03-29T05:05:37.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:37 smithi181 ceph-mon[101970]: failed to encode map e723 with expected crc 2024-03-29T05:05:37.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:37 smithi181 ceph-mon[101970]: failed to encode map e723 with expected crc 2024-03-29T05:05:37.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:37 smithi181 ceph-mon[101970]: osdmap e724: 8 total, 8 up, 8 in 2024-03-29T05:05:37.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:37 smithi181 ceph-mon[101970]: failed to encode map e724 with expected crc 2024-03-29T05:05:37.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:37 smithi181 ceph-mon[106466]: failed to encode map e723 with expected crc 2024-03-29T05:05:37.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:37 smithi181 ceph-mon[106466]: failed to encode map e723 with expected crc 2024-03-29T05:05:37.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:37 smithi181 ceph-mon[106466]: osdmap e724: 8 total, 8 up, 8 in 2024-03-29T05:05:37.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:37 smithi181 ceph-mon[106466]: failed to encode map e724 with expected crc 2024-03-29T05:05:38.278 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ObjectMapConsistentSnap (10430 ms) 2024-03-29T05:05:38.278 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.Metadata 2024-03-29T05:05:38.279 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:38.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:38 smithi187 ceph-mon[93536]: failed to encode map e724 with expected crc 2024-03-29T05:05:38.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:38 smithi187 ceph-mon[93536]: failed to encode map e724 with expected crc 2024-03-29T05:05:38.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:38 smithi187 ceph-mon[93536]: failed to encode map e724 with expected crc 2024-03-29T05:05:38.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:38 smithi187 ceph-mon[93536]: pgmap v792: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 3.5 KiB/s wr, 89 op/s 2024-03-29T05:05:38.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:38 smithi187 ceph-mon[93536]: osdmap e725: 8 total, 8 up, 8 in 2024-03-29T05:05:38.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:38 smithi187 ceph-mon[93536]: failed to encode map e725 with expected crc 2024-03-29T05:05:38.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:38 smithi181 ceph-mon[106466]: failed to encode map e724 with expected crc 2024-03-29T05:05:38.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:38 smithi181 ceph-mon[106466]: failed to encode map e724 with expected crc 2024-03-29T05:05:38.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:38 smithi181 ceph-mon[106466]: failed to encode map e724 with expected crc 2024-03-29T05:05:38.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:38 smithi181 ceph-mon[106466]: pgmap v792: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 3.5 KiB/s wr, 89 op/s 2024-03-29T05:05:38.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:38 smithi181 ceph-mon[106466]: osdmap e725: 8 total, 8 up, 8 in 2024-03-29T05:05:38.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:38 smithi181 ceph-mon[106466]: failed to encode map e725 with expected crc 2024-03-29T05:05:38.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:38 smithi181 ceph-mon[101970]: failed to encode map e724 with expected crc 2024-03-29T05:05:38.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:38 smithi181 ceph-mon[101970]: failed to encode map e724 with expected crc 2024-03-29T05:05:38.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:38 smithi181 ceph-mon[101970]: failed to encode map e724 with expected crc 2024-03-29T05:05:38.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:38 smithi181 ceph-mon[101970]: pgmap v792: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 3.5 KiB/s wr, 89 op/s 2024-03-29T05:05:38.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:38 smithi181 ceph-mon[101970]: osdmap e725: 8 total, 8 up, 8 in 2024-03-29T05:05:38.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:38 smithi181 ceph-mon[101970]: failed to encode map e725 with expected crc 2024-03-29T05:05:38.779 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.Metadata (501 ms) 2024-03-29T05:05:38.780 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.MetadataPP 2024-03-29T05:05:38.780 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:39.206 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:05:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:05:38] "GET /metrics HTTP/1.1" 200 74969 "" "Prometheus/2.43.0" 2024-03-29T05:05:39.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:39 smithi187 ceph-mon[93536]: failed to encode map e725 with expected crc 2024-03-29T05:05:39.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:39 smithi187 ceph-mon[93536]: failed to encode map e725 with expected crc 2024-03-29T05:05:39.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:39 smithi187 ceph-mon[93536]: osdmap e726: 8 total, 8 up, 8 in 2024-03-29T05:05:39.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:39 smithi187 ceph-mon[93536]: failed to encode map e726 with expected crc 2024-03-29T05:05:39.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:39 smithi181 ceph-mon[106466]: failed to encode map e725 with expected crc 2024-03-29T05:05:39.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:39 smithi181 ceph-mon[106466]: failed to encode map e725 with expected crc 2024-03-29T05:05:39.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:39 smithi181 ceph-mon[106466]: osdmap e726: 8 total, 8 up, 8 in 2024-03-29T05:05:39.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:39 smithi181 ceph-mon[106466]: failed to encode map e726 with expected crc 2024-03-29T05:05:39.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:39 smithi181 ceph-mon[101970]: failed to encode map e725 with expected crc 2024-03-29T05:05:39.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:39 smithi181 ceph-mon[101970]: failed to encode map e725 with expected crc 2024-03-29T05:05:39.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:39 smithi181 ceph-mon[101970]: osdmap e726: 8 total, 8 up, 8 in 2024-03-29T05:05:39.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:39 smithi181 ceph-mon[101970]: failed to encode map e726 with expected crc 2024-03-29T05:05:39.781 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.MetadataPP (1002 ms) 2024-03-29T05:05:39.781 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.UpdateFeatures 2024-03-29T05:05:39.781 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:39.875 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.UpdateFeatures (94 ms) 2024-03-29T05:05:39.876 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.FeaturesBitmaskString 2024-03-29T05:05:39.876 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.FeaturesBitmaskString (0 ms) 2024-03-29T05:05:39.876 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.RebuildObjectMap 2024-03-29T05:05:39.876 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:40.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:40 smithi187 ceph-mon[93536]: failed to encode map e726 with expected crc 2024-03-29T05:05:40.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:40 smithi187 ceph-mon[93536]: failed to encode map e726 with expected crc 2024-03-29T05:05:40.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:40 smithi187 ceph-mon[93536]: pgmap v795: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 KiB/s rd, 12 KiB/s wr, 98 op/s 2024-03-29T05:05:40.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:40 smithi187 ceph-mon[93536]: osdmap e727: 8 total, 8 up, 8 in 2024-03-29T05:05:40.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:40 smithi181 ceph-mon[101970]: failed to encode map e726 with expected crc 2024-03-29T05:05:40.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:40 smithi181 ceph-mon[101970]: failed to encode map e726 with expected crc 2024-03-29T05:05:40.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:40 smithi181 ceph-mon[101970]: pgmap v795: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 KiB/s rd, 12 KiB/s wr, 98 op/s 2024-03-29T05:05:40.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:40 smithi181 ceph-mon[101970]: osdmap e727: 8 total, 8 up, 8 in 2024-03-29T05:05:40.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:40 smithi181 ceph-mon[106466]: failed to encode map e726 with expected crc 2024-03-29T05:05:40.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:40 smithi181 ceph-mon[106466]: failed to encode map e726 with expected crc 2024-03-29T05:05:40.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:40 smithi181 ceph-mon[106466]: pgmap v795: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 KiB/s rd, 12 KiB/s wr, 98 op/s 2024-03-29T05:05:40.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:40 smithi181 ceph-mon[106466]: osdmap e727: 8 total, 8 up, 8 in 2024-03-29T05:05:40.751 INFO:tasks.workunit.client.0.smithi181.stdout:0.00% done 2024-03-29T05:05:40.751 INFO:tasks.workunit.client.0.smithi181.stdout:25.00% done 2024-03-29T05:05:40.751 INFO:tasks.workunit.client.0.smithi181.stdout:50.00% done 2024-03-29T05:05:40.751 INFO:tasks.workunit.client.0.smithi181.stdout:75.00% done 2024-03-29T05:05:40.751 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.RebuildObjectMap (876 ms) 2024-03-29T05:05:40.751 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.RebuildNewObjectMap 2024-03-29T05:05:40.782 INFO:tasks.workunit.client.0.smithi181.stdout:0.00% done 2024-03-29T05:05:40.783 INFO:tasks.workunit.client.0.smithi181.stdout:25.00% done 2024-03-29T05:05:40.783 INFO:tasks.workunit.client.0.smithi181.stdout:50.00% done 2024-03-29T05:05:40.783 INFO:tasks.workunit.client.0.smithi181.stdout:75.00% done 2024-03-29T05:05:40.783 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.RebuildNewObjectMap (31 ms) 2024-03-29T05:05:40.783 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.CheckObjectMap 2024-03-29T05:05:40.783 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:41.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:41 smithi187 ceph-mon[93536]: failed to encode map e727 with expected crc 2024-03-29T05:05:41.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:41 smithi187 ceph-mon[93536]: failed to encode map e727 with expected crc 2024-03-29T05:05:41.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:41 smithi187 ceph-mon[93536]: osdmap e728: 8 total, 8 up, 8 in 2024-03-29T05:05:41.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:05:41.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:41 smithi181 ceph-mon[101970]: failed to encode map e727 with expected crc 2024-03-29T05:05:41.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:41 smithi181 ceph-mon[101970]: failed to encode map e727 with expected crc 2024-03-29T05:05:41.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:41 smithi181 ceph-mon[101970]: osdmap e728: 8 total, 8 up, 8 in 2024-03-29T05:05:41.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:05:41.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:41 smithi181 ceph-mon[106466]: failed to encode map e727 with expected crc 2024-03-29T05:05:41.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:41 smithi181 ceph-mon[106466]: failed to encode map e727 with expected crc 2024-03-29T05:05:41.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:41 smithi181 ceph-mon[106466]: osdmap e728: 8 total, 8 up, 8 in 2024-03-29T05:05:41.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:05:41.761 INFO:tasks.workunit.client.0.smithi181.stdout:0.00% done 2024-03-29T05:05:41.761 INFO:tasks.workunit.client.0.smithi181.stdout:25.00% done 2024-03-29T05:05:41.761 INFO:tasks.workunit.client.0.smithi181.stdout:50.00% done 2024-03-29T05:05:41.761 INFO:tasks.workunit.client.0.smithi181.stdout:75.00% done 2024-03-29T05:05:41.761 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.CheckObjectMap (979 ms) 2024-03-29T05:05:41.762 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.BlockingAIO 2024-03-29T05:05:41.762 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:41.787 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.BlockingAIO (26 ms) 2024-03-29T05:05:41.787 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ExclusiveLockTransition 2024-03-29T05:05:41.787 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:41.925 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ExclusiveLockTransition (138 ms) 2024-03-29T05:05:41.926 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ExclusiveLockReadTransition 2024-03-29T05:05:41.926 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T05:05:41.926 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ExclusiveLockReadTransition (0 ms) 2024-03-29T05:05:41.926 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.CacheMayCopyOnWrite 2024-03-29T05:05:41.926 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:42.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:42 smithi187 ceph-mon[93536]: failed to encode map e728 with expected crc 2024-03-29T05:05:42.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:42 smithi187 ceph-mon[93536]: failed to encode map e728 with expected crc 2024-03-29T05:05:42.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:42 smithi187 ceph-mon[93536]: pgmap v798: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 8.2 KiB/s wr, 8 op/s 2024-03-29T05:05:42.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:42 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:05:42.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:42 smithi187 ceph-mon[93536]: osdmap e729: 8 total, 8 up, 8 in 2024-03-29T05:05:42.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:42 smithi187 ceph-mon[93536]: failed to encode map e729 with expected crc 2024-03-29T05:05:42.638 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:42 smithi181 ceph-mon[101970]: failed to encode map e728 with expected crc 2024-03-29T05:05:42.638 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:42 smithi181 ceph-mon[101970]: failed to encode map e728 with expected crc 2024-03-29T05:05:42.638 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:42 smithi181 ceph-mon[101970]: pgmap v798: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 8.2 KiB/s wr, 8 op/s 2024-03-29T05:05:42.638 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:42 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:05:42.638 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:42 smithi181 ceph-mon[101970]: osdmap e729: 8 total, 8 up, 8 in 2024-03-29T05:05:42.638 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:42 smithi181 ceph-mon[101970]: failed to encode map e729 with expected crc 2024-03-29T05:05:42.638 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:42 smithi181 ceph-mon[106466]: failed to encode map e728 with expected crc 2024-03-29T05:05:42.639 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:42 smithi181 ceph-mon[106466]: failed to encode map e728 with expected crc 2024-03-29T05:05:42.639 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:42 smithi181 ceph-mon[106466]: pgmap v798: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 8.2 KiB/s wr, 8 op/s 2024-03-29T05:05:42.639 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:42 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:05:42.639 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:42 smithi181 ceph-mon[106466]: osdmap e729: 8 total, 8 up, 8 in 2024-03-29T05:05:42.639 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:42 smithi181 ceph-mon[106466]: failed to encode map e729 with expected crc 2024-03-29T05:05:42.770 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.CacheMayCopyOnWrite (845 ms) 2024-03-29T05:05:42.770 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot 2024-03-29T05:05:42.770 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:43.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:43 smithi187 ceph-mon[93536]: failed to encode map e729 with expected crc 2024-03-29T05:05:43.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:43 smithi187 ceph-mon[93536]: failed to encode map e729 with expected crc 2024-03-29T05:05:43.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:43 smithi187 ceph-mon[93536]: osdmap e730: 8 total, 8 up, 8 in 2024-03-29T05:05:43.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:43 smithi181 ceph-mon[101970]: failed to encode map e729 with expected crc 2024-03-29T05:05:43.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:43 smithi181 ceph-mon[101970]: failed to encode map e729 with expected crc 2024-03-29T05:05:43.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:43 smithi181 ceph-mon[101970]: osdmap e730: 8 total, 8 up, 8 in 2024-03-29T05:05:43.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:43 smithi181 ceph-mon[106466]: failed to encode map e729 with expected crc 2024-03-29T05:05:43.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:43 smithi181 ceph-mon[106466]: failed to encode map e729 with expected crc 2024-03-29T05:05:43.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:43 smithi181 ceph-mon[106466]: osdmap e730: 8 total, 8 up, 8 in 2024-03-29T05:05:43.719 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot (949 ms) 2024-03-29T05:05:43.719 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestImageOptions 2024-03-29T05:05:44.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:44 smithi187 ceph-mon[93536]: failed to encode map e730 with expected crc 2024-03-29T05:05:44.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:44 smithi187 ceph-mon[93536]: failed to encode map e730 with expected crc 2024-03-29T05:05:44.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:44 smithi187 ceph-mon[93536]: pgmap v801: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 770 KiB/s rd, 252 KiB/s wr, 1.15k op/s 2024-03-29T05:05:44.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:44 smithi187 ceph-mon[93536]: osdmap e731: 8 total, 8 up, 8 in 2024-03-29T05:05:44.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:44 smithi187 ceph-mon[93536]: failed to encode map e731 with expected crc 2024-03-29T05:05:44.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:44 smithi181 ceph-mon[101970]: failed to encode map e730 with expected crc 2024-03-29T05:05:44.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:44 smithi181 ceph-mon[101970]: failed to encode map e730 with expected crc 2024-03-29T05:05:44.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:44 smithi181 ceph-mon[101970]: pgmap v801: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 770 KiB/s rd, 252 KiB/s wr, 1.15k op/s 2024-03-29T05:05:44.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:44 smithi181 ceph-mon[101970]: osdmap e731: 8 total, 8 up, 8 in 2024-03-29T05:05:44.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:44 smithi181 ceph-mon[101970]: failed to encode map e731 with expected crc 2024-03-29T05:05:44.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:44 smithi181 ceph-mon[106466]: failed to encode map e730 with expected crc 2024-03-29T05:05:44.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:44 smithi181 ceph-mon[106466]: failed to encode map e730 with expected crc 2024-03-29T05:05:44.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:44 smithi181 ceph-mon[106466]: pgmap v801: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 770 KiB/s rd, 252 KiB/s wr, 1.15k op/s 2024-03-29T05:05:44.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:44 smithi181 ceph-mon[106466]: osdmap e731: 8 total, 8 up, 8 in 2024-03-29T05:05:44.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:44 smithi181 ceph-mon[106466]: failed to encode map e731 with expected crc 2024-03-29T05:05:44.823 INFO:tasks.workunit.client.0.smithi181.stdout:0.00% done 2024-03-29T05:05:44.824 INFO:tasks.workunit.client.0.smithi181.stdout:100.00% done 2024-03-29T05:05:44.824 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestImageOptions (1104 ms) 2024-03-29T05:05:44.824 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestImageOptionsPP 2024-03-29T05:05:45.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:45 smithi181 ceph-mon[106466]: failed to encode map e731 with expected crc 2024-03-29T05:05:45.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:45 smithi181 ceph-mon[106466]: failed to encode map e731 with expected crc 2024-03-29T05:05:45.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:45 smithi181 ceph-mon[106466]: osdmap e732: 8 total, 8 up, 8 in 2024-03-29T05:05:45.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:45 smithi181 ceph-mon[106466]: failed to encode map e732 with expected crc 2024-03-29T05:05:45.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:45 smithi181 ceph-mon[106466]: failed to encode map e732 with expected crc 2024-03-29T05:05:45.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:45 smithi181 ceph-mon[101970]: failed to encode map e731 with expected crc 2024-03-29T05:05:45.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:45 smithi181 ceph-mon[101970]: failed to encode map e731 with expected crc 2024-03-29T05:05:45.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:45 smithi181 ceph-mon[101970]: osdmap e732: 8 total, 8 up, 8 in 2024-03-29T05:05:45.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:45 smithi181 ceph-mon[101970]: failed to encode map e732 with expected crc 2024-03-29T05:05:45.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:45 smithi181 ceph-mon[101970]: failed to encode map e732 with expected crc 2024-03-29T05:05:45.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:45 smithi187 ceph-mon[93536]: failed to encode map e731 with expected crc 2024-03-29T05:05:45.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:45 smithi187 ceph-mon[93536]: failed to encode map e731 with expected crc 2024-03-29T05:05:45.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:45 smithi187 ceph-mon[93536]: osdmap e732: 8 total, 8 up, 8 in 2024-03-29T05:05:45.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:45 smithi187 ceph-mon[93536]: failed to encode map e732 with expected crc 2024-03-29T05:05:45.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:45 smithi187 ceph-mon[93536]: failed to encode map e732 with expected crc 2024-03-29T05:05:45.814 INFO:tasks.workunit.client.0.smithi181.stdout:0.00% done 2024-03-29T05:05:45.815 INFO:tasks.workunit.client.0.smithi181.stdout:100.00% done 2024-03-29T05:05:45.815 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestImageOptionsPP (991 ms) 2024-03-29T05:05:45.815 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.EventSocketPipe 2024-03-29T05:05:45.815 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.EventSocketPipe (0 ms) 2024-03-29T05:05:45.815 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.EventSocketEventfd 2024-03-29T05:05:45.815 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.EventSocketEventfd (0 ms) 2024-03-29T05:05:45.815 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ImagePollIO 2024-03-29T05:05:45.815 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:45.858 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:05:45.859 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:05:45.859 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:05:45.859 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:05:45.859 INFO:tasks.workunit.client.0.smithi181.stdout:wrote: 512 2024-03-29T05:05:45.859 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:05:45.859 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:05:45.859 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:05:45.859 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:05:45.859 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:05:45.859 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:05:45.859 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:05:45.859 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:05:45.860 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:05:45.860 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:05:45.860 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:05:45.860 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:05:45.860 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:05:45.860 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:05:45.860 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:05:45.860 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:05:45.860 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:05:45.860 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:05:45.860 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:05:45.860 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:05:45.860 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:05:45.860 INFO:tasks.workunit.client.0.smithi181.stdout:started write 2024-03-29T05:05:45.861 INFO:tasks.workunit.client.0.smithi181.stdout:write completion cb called! 2024-03-29T05:05:45.861 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 0 2024-03-29T05:05:45.861 INFO:tasks.workunit.client.0.smithi181.stdout:finished write 2024-03-29T05:05:45.861 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:05:45.861 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:05:45.861 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:05:45.861 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:05:45.861 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:05:45.861 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:05:45.861 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:05:45.861 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:05:45.861 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:05:45.861 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:05:45.861 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:05:45.862 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:05:45.862 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:05:45.862 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:05:45.862 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:05:45.862 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:05:45.862 INFO:tasks.workunit.client.0.smithi181.stdout:created completion 2024-03-29T05:05:45.862 INFO:tasks.workunit.client.0.smithi181.stdout:started read 2024-03-29T05:05:45.862 INFO:tasks.workunit.client.0.smithi181.stdout:read completion cb called! 2024-03-29T05:05:45.862 INFO:tasks.workunit.client.0.smithi181.stdout:return value is: 512 2024-03-29T05:05:45.862 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ImagePollIO (44 ms) 2024-03-29T05:05:45.862 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.Mirror 2024-03-29T05:05:45.863 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:45.871 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:46.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:46 smithi187 ceph-mon[93536]: failed to encode map e732 with expected crc 2024-03-29T05:05:46.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:46 smithi187 ceph-mon[93536]: failed to encode map e732 with expected crc 2024-03-29T05:05:46.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:46 smithi187 ceph-mon[93536]: pgmap v804: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 770 KiB/s rd, 252 KiB/s wr, 1.15k op/s 2024-03-29T05:05:46.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:46 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:46.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:46 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:46.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:46 smithi187 ceph-mon[93536]: osdmap e733: 8 total, 8 up, 8 in 2024-03-29T05:05:46.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:46 smithi187 ceph-mon[93536]: failed to encode map e733 with expected crc 2024-03-29T05:05:46.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:46 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:46.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:46 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:46.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[106466]: failed to encode map e732 with expected crc 2024-03-29T05:05:46.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[106466]: failed to encode map e732 with expected crc 2024-03-29T05:05:46.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[106466]: pgmap v804: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 770 KiB/s rd, 252 KiB/s wr, 1.15k op/s 2024-03-29T05:05:46.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:46.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:46.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[106466]: osdmap e733: 8 total, 8 up, 8 in 2024-03-29T05:05:46.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[106466]: failed to encode map e733 with expected crc 2024-03-29T05:05:46.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:46.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:46.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[101970]: failed to encode map e732 with expected crc 2024-03-29T05:05:46.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[101970]: failed to encode map e732 with expected crc 2024-03-29T05:05:46.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[101970]: pgmap v804: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 770 KiB/s rd, 252 KiB/s wr, 1.15k op/s 2024-03-29T05:05:46.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:46.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:46.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[101970]: osdmap e733: 8 total, 8 up, 8 in 2024-03-29T05:05:46.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[101970]: failed to encode map e733 with expected crc 2024-03-29T05:05:46.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:46.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:46 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:46.993 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.Mirror (1135 ms) 2024-03-29T05:05:46.994 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.MirrorPeerAttributes 2024-03-29T05:05:47.013 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.MirrorPeerAttributes (20 ms) 2024-03-29T05:05:47.013 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.CreateWithMirrorEnabled 2024-03-29T05:05:47.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: failed to encode map e733 with expected crc 2024-03-29T05:05:47.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: failed to encode map e733 with expected crc 2024-03-29T05:05:47.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:05:47.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:05:47.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: pgmap v806: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 82 KiB/s rd, 15 KiB/s wr, 108 op/s 2024-03-29T05:05:47.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: pgmap v807: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:05:47.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:47.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: failed to encode map e734 with expected crc 2024-03-29T05:05:47.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: osdmap e734: 8 total, 8 up, 8 in 2024-03-29T05:05:47.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-29T05:05:47.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-29T05:05:47.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-03-29T05:05:47.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/a735d34b-9aed-4dad-8855-8a18373b7056"}]: dispatch 2024-03-29T05:05:47.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/a735d34b-9aed-4dad-8855-8a18373b7056"}]: dispatch 2024-03-29T05:05:47.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/a735d34b-9aed-4dad-8855-8a18373b7056"}]': finished 2024-03-29T05:05:47.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/3af36727-d605-46a0-90bb-efda12294d1b"}]: dispatch 2024-03-29T05:05:47.637 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/3af36727-d605-46a0-90bb-efda12294d1b"}]: dispatch 2024-03-29T05:05:47.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/3af36727-d605-46a0-90bb-efda12294d1b"}]': finished 2024-03-29T05:05:47.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/eec13c83-5308-4ccf-9fc9-49dc1193cd63"}]: dispatch 2024-03-29T05:05:47.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/eec13c83-5308-4ccf-9fc9-49dc1193cd63"}]: dispatch 2024-03-29T05:05:47.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/eec13c83-5308-4ccf-9fc9-49dc1193cd63"}]': finished 2024-03-29T05:05:47.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/ecd9c016-af50-4957-8938-886c521e5a6c"}]: dispatch 2024-03-29T05:05:47.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' 2024-03-29T05:05:47.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/ecd9c016-af50-4957-8938-886c521e5a6c"}]: dispatch 2024-03-29T05:05:47.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/ecd9c016-af50-4957-8938-886c521e5a6c"}]: dispatch 2024-03-29T05:05:47.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/ecd9c016-af50-4957-8938-886c521e5a6c"}]: dispatch 2024-03-29T05:05:47.638 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:47 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/ecd9c016-af50-4957-8938-886c521e5a6c"}]': finished 2024-03-29T05:05:47.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: failed to encode map e733 with expected crc 2024-03-29T05:05:47.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: failed to encode map e733 with expected crc 2024-03-29T05:05:47.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:05:47.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:05:47.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: pgmap v806: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 82 KiB/s rd, 15 KiB/s wr, 108 op/s 2024-03-29T05:05:47.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: pgmap v807: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:05:47.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:47.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: failed to encode map e734 with expected crc 2024-03-29T05:05:47.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: osdmap e734: 8 total, 8 up, 8 in 2024-03-29T05:05:47.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-29T05:05:47.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-29T05:05:47.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-03-29T05:05:47.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/a735d34b-9aed-4dad-8855-8a18373b7056"}]: dispatch 2024-03-29T05:05:47.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/a735d34b-9aed-4dad-8855-8a18373b7056"}]: dispatch 2024-03-29T05:05:47.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/a735d34b-9aed-4dad-8855-8a18373b7056"}]': finished 2024-03-29T05:05:47.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/3af36727-d605-46a0-90bb-efda12294d1b"}]: dispatch 2024-03-29T05:05:47.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/3af36727-d605-46a0-90bb-efda12294d1b"}]: dispatch 2024-03-29T05:05:47.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/3af36727-d605-46a0-90bb-efda12294d1b"}]': finished 2024-03-29T05:05:47.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/eec13c83-5308-4ccf-9fc9-49dc1193cd63"}]: dispatch 2024-03-29T05:05:47.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/eec13c83-5308-4ccf-9fc9-49dc1193cd63"}]: dispatch 2024-03-29T05:05:47.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/eec13c83-5308-4ccf-9fc9-49dc1193cd63"}]': finished 2024-03-29T05:05:47.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/ecd9c016-af50-4957-8938-886c521e5a6c"}]: dispatch 2024-03-29T05:05:47.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' 2024-03-29T05:05:47.708 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/ecd9c016-af50-4957-8938-886c521e5a6c"}]: dispatch 2024-03-29T05:05:47.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: failed to encode map e733 with expected crc 2024-03-29T05:05:47.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: failed to encode map e733 with expected crc 2024-03-29T05:05:47.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:05:47.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:05:47.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: pgmap v806: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 82 KiB/s rd, 15 KiB/s wr, 108 op/s 2024-03-29T05:05:47.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: pgmap v807: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:05:47.708 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:05:47.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: failed to encode map e734 with expected crc 2024-03-29T05:05:47.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: osdmap e734: 8 total, 8 up, 8 in 2024-03-29T05:05:47.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-29T05:05:47.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-29T05:05:47.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-03-29T05:05:47.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/a735d34b-9aed-4dad-8855-8a18373b7056"}]: dispatch 2024-03-29T05:05:47.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/a735d34b-9aed-4dad-8855-8a18373b7056"}]: dispatch 2024-03-29T05:05:47.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/a735d34b-9aed-4dad-8855-8a18373b7056"}]': finished 2024-03-29T05:05:47.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/3af36727-d605-46a0-90bb-efda12294d1b"}]: dispatch 2024-03-29T05:05:47.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/3af36727-d605-46a0-90bb-efda12294d1b"}]: dispatch 2024-03-29T05:05:47.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/3af36727-d605-46a0-90bb-efda12294d1b"}]': finished 2024-03-29T05:05:47.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/eec13c83-5308-4ccf-9fc9-49dc1193cd63"}]: dispatch 2024-03-29T05:05:47.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/eec13c83-5308-4ccf-9fc9-49dc1193cd63"}]: dispatch 2024-03-29T05:05:47.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/eec13c83-5308-4ccf-9fc9-49dc1193cd63"}]': finished 2024-03-29T05:05:47.709 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/ecd9c016-af50-4957-8938-886c521e5a6c"}]: dispatch 2024-03-29T05:05:47.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' 2024-03-29T05:05:47.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/ecd9c016-af50-4957-8938-886c521e5a6c"}]: dispatch 2024-03-29T05:05:47.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/ecd9c016-af50-4957-8938-886c521e5a6c"}]: dispatch 2024-03-29T05:05:47.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/ecd9c016-af50-4957-8938-886c521e5a6c"}]: dispatch 2024-03-29T05:05:47.710 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/ecd9c016-af50-4957-8938-886c521e5a6c"}]': finished 2024-03-29T05:05:47.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3557785454' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/ecd9c016-af50-4957-8938-886c521e5a6c"}]: dispatch 2024-03-29T05:05:47.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/ecd9c016-af50-4957-8938-886c521e5a6c"}]: dispatch 2024-03-29T05:05:47.710 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:47 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/ecd9c016-af50-4957-8938-886c521e5a6c"}]': finished 2024-03-29T05:05:48.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:48 smithi187 ceph-mon[93536]: failed to encode map e734 with expected crc 2024-03-29T05:05:48.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:48 smithi187 ceph-mon[93536]: failed to encode map e734 with expected crc 2024-03-29T05:05:48.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:48 smithi187 ceph-mon[93536]: osdmap e735: 8 total, 8 up, 8 in 2024-03-29T05:05:48.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:48 smithi181 ceph-mon[101970]: failed to encode map e734 with expected crc 2024-03-29T05:05:48.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:48 smithi181 ceph-mon[101970]: failed to encode map e734 with expected crc 2024-03-29T05:05:48.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:48 smithi181 ceph-mon[101970]: osdmap e735: 8 total, 8 up, 8 in 2024-03-29T05:05:48.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:48 smithi181 ceph-mon[106466]: failed to encode map e734 with expected crc 2024-03-29T05:05:48.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:48 smithi181 ceph-mon[106466]: failed to encode map e734 with expected crc 2024-03-29T05:05:48.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:48 smithi181 ceph-mon[106466]: osdmap e735: 8 total, 8 up, 8 in 2024-03-29T05:05:49.380 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:05:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:05:48] "GET /metrics HTTP/1.1" 200 74972 "" "Prometheus/2.43.0" 2024-03-29T05:05:49.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:49 smithi187 ceph-mon[93536]: failed to encode map e735 with expected crc 2024-03-29T05:05:49.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:49 smithi187 ceph-mon[93536]: failed to encode map e735 with expected crc 2024-03-29T05:05:49.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:49 smithi187 ceph-mon[93536]: pgmap v810: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 498 KiB/s rd, 2.8 MiB/s wr, 675 op/s 2024-03-29T05:05:49.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:49 smithi187 ceph-mon[93536]: osdmap e736: 8 total, 8 up, 8 in 2024-03-29T05:05:49.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:49 smithi181 ceph-mon[101970]: failed to encode map e735 with expected crc 2024-03-29T05:05:49.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:49 smithi181 ceph-mon[101970]: failed to encode map e735 with expected crc 2024-03-29T05:05:49.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:49 smithi181 ceph-mon[101970]: pgmap v810: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 498 KiB/s rd, 2.8 MiB/s wr, 675 op/s 2024-03-29T05:05:49.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:49 smithi181 ceph-mon[101970]: osdmap e736: 8 total, 8 up, 8 in 2024-03-29T05:05:49.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:49 smithi181 ceph-mon[106466]: failed to encode map e735 with expected crc 2024-03-29T05:05:49.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:49 smithi181 ceph-mon[106466]: failed to encode map e735 with expected crc 2024-03-29T05:05:49.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:49 smithi181 ceph-mon[106466]: pgmap v810: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 498 KiB/s rd, 2.8 MiB/s wr, 675 op/s 2024-03-29T05:05:49.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:49 smithi181 ceph-mon[106466]: osdmap e736: 8 total, 8 up, 8 in 2024-03-29T05:05:50.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:50 smithi187 ceph-mon[93536]: failed to encode map e736 with expected crc 2024-03-29T05:05:50.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:50 smithi187 ceph-mon[93536]: failed to encode map e736 with expected crc 2024-03-29T05:05:50.636 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:50 smithi187 ceph-mon[93536]: osdmap e737: 8 total, 8 up, 8 in 2024-03-29T05:05:50.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:50 smithi181 ceph-mon[101970]: failed to encode map e736 with expected crc 2024-03-29T05:05:50.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:50 smithi181 ceph-mon[101970]: failed to encode map e736 with expected crc 2024-03-29T05:05:50.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:50 smithi181 ceph-mon[101970]: osdmap e737: 8 total, 8 up, 8 in 2024-03-29T05:05:50.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:50 smithi181 ceph-mon[106466]: failed to encode map e736 with expected crc 2024-03-29T05:05:50.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:50 smithi181 ceph-mon[106466]: failed to encode map e736 with expected crc 2024-03-29T05:05:50.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:50 smithi181 ceph-mon[106466]: osdmap e737: 8 total, 8 up, 8 in 2024-03-29T05:05:51.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:51 smithi181 ceph-mon[106466]: failed to encode map e737 with expected crc 2024-03-29T05:05:51.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:51 smithi181 ceph-mon[106466]: failed to encode map e737 with expected crc 2024-03-29T05:05:51.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:51 smithi181 ceph-mon[106466]: pgmap v813: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 369 KiB/s rd, 2.1 MiB/s wr, 499 op/s 2024-03-29T05:05:51.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:51 smithi181 ceph-mon[106466]: osdmap e738: 8 total, 8 up, 8 in 2024-03-29T05:05:51.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:51 smithi181 ceph-mon[101970]: failed to encode map e737 with expected crc 2024-03-29T05:05:51.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:51 smithi181 ceph-mon[101970]: failed to encode map e737 with expected crc 2024-03-29T05:05:51.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:51 smithi181 ceph-mon[101970]: pgmap v813: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 369 KiB/s rd, 2.1 MiB/s wr, 499 op/s 2024-03-29T05:05:51.707 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:51 smithi181 ceph-mon[101970]: osdmap e738: 8 total, 8 up, 8 in 2024-03-29T05:05:51.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:51 smithi187 ceph-mon[93536]: failed to encode map e737 with expected crc 2024-03-29T05:05:51.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:51 smithi187 ceph-mon[93536]: failed to encode map e737 with expected crc 2024-03-29T05:05:51.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:51 smithi187 ceph-mon[93536]: pgmap v813: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 369 KiB/s rd, 2.1 MiB/s wr, 499 op/s 2024-03-29T05:05:51.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:51 smithi187 ceph-mon[93536]: osdmap e738: 8 total, 8 up, 8 in 2024-03-29T05:05:52.066 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.CreateWithMirrorEnabled (5053 ms) 2024-03-29T05:05:52.067 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot 2024-03-29T05:05:52.067 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:52.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:52 smithi181 ceph-mon[101970]: failed to encode map e738 with expected crc 2024-03-29T05:05:52.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:52 smithi181 ceph-mon[101970]: failed to encode map e738 with expected crc 2024-03-29T05:05:52.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:52 smithi181 ceph-mon[101970]: failed to encode map e738 with expected crc 2024-03-29T05:05:52.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:52 smithi181 ceph-mon[101970]: failed to encode map e738 with expected crc 2024-03-29T05:05:52.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:52 smithi181 ceph-mon[101970]: osdmap e739: 8 total, 8 up, 8 in 2024-03-29T05:05:52.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:52 smithi181 ceph-mon[106466]: failed to encode map e738 with expected crc 2024-03-29T05:05:52.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:52 smithi181 ceph-mon[106466]: failed to encode map e738 with expected crc 2024-03-29T05:05:52.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:52 smithi181 ceph-mon[106466]: failed to encode map e738 with expected crc 2024-03-29T05:05:52.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:52 smithi181 ceph-mon[106466]: failed to encode map e738 with expected crc 2024-03-29T05:05:52.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:52 smithi181 ceph-mon[106466]: osdmap e739: 8 total, 8 up, 8 in 2024-03-29T05:05:52.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:52 smithi187 ceph-mon[93536]: failed to encode map e738 with expected crc 2024-03-29T05:05:52.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:52 smithi187 ceph-mon[93536]: failed to encode map e738 with expected crc 2024-03-29T05:05:52.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:52 smithi187 ceph-mon[93536]: failed to encode map e738 with expected crc 2024-03-29T05:05:52.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:52 smithi187 ceph-mon[93536]: failed to encode map e738 with expected crc 2024-03-29T05:05:52.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:52 smithi187 ceph-mon[93536]: osdmap e739: 8 total, 8 up, 8 in 2024-03-29T05:05:53.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:53 smithi181 ceph-mon[101970]: failed to encode map e739 with expected crc 2024-03-29T05:05:53.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:53 smithi181 ceph-mon[101970]: failed to encode map e739 with expected crc 2024-03-29T05:05:53.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:53 smithi181 ceph-mon[101970]: pgmap v816: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 161 KiB/s rd, 74 KiB/s wr, 208 op/s 2024-03-29T05:05:53.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:53 smithi181 ceph-mon[101970]: osdmap e740: 8 total, 8 up, 8 in 2024-03-29T05:05:53.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:53 smithi181 ceph-mon[106466]: failed to encode map e739 with expected crc 2024-03-29T05:05:53.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:53 smithi181 ceph-mon[106466]: failed to encode map e739 with expected crc 2024-03-29T05:05:53.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:53 smithi181 ceph-mon[106466]: pgmap v816: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 161 KiB/s rd, 74 KiB/s wr, 208 op/s 2024-03-29T05:05:53.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:53 smithi181 ceph-mon[106466]: osdmap e740: 8 total, 8 up, 8 in 2024-03-29T05:05:53.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:53 smithi187 ceph-mon[93536]: failed to encode map e739 with expected crc 2024-03-29T05:05:53.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:53 smithi187 ceph-mon[93536]: failed to encode map e739 with expected crc 2024-03-29T05:05:53.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:53 smithi187 ceph-mon[93536]: pgmap v816: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 161 KiB/s rd, 74 KiB/s wr, 208 op/s 2024-03-29T05:05:53.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:53 smithi187 ceph-mon[93536]: osdmap e740: 8 total, 8 up, 8 in 2024-03-29T05:05:54.056 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot (1989 ms) 2024-03-29T05:05:54.056 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ExclusiveLock 2024-03-29T05:05:54.056 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:54.111 INFO:tasks.workunit.client.0.smithi181.stdout:1: acquiring exclusive lock 2024-03-29T05:05:54.111 INFO:tasks.workunit.client.0.smithi181.stdout:2: acquiring exclusive lock 2024-03-29T05:05:54.114 INFO:tasks.workunit.client.0.smithi181.stdout:2: exclusive lock acquired 2024-03-29T05:05:54.137 INFO:tasks.workunit.client.0.smithi181.stdout:2: releasing exclusive lock 2024-03-29T05:05:54.140 INFO:tasks.workunit.client.0.smithi181.stdout:2: exclusive lock released 2024-03-29T05:05:54.141 INFO:tasks.workunit.client.0.smithi181.stdout:2: acquiring exclusive lock 2024-03-29T05:05:54.144 INFO:tasks.workunit.client.0.smithi181.stdout:2: exclusive lock acquired 2024-03-29T05:05:54.188 INFO:tasks.workunit.client.0.smithi181.stdout:2: releasing exclusive lock 2024-03-29T05:05:54.190 INFO:tasks.workunit.client.0.smithi181.stdout:2: exclusive lock released 2024-03-29T05:05:54.190 INFO:tasks.workunit.client.0.smithi181.stdout:2: acquiring exclusive lock 2024-03-29T05:05:54.194 INFO:tasks.workunit.client.0.smithi181.stdout:2: exclusive lock acquired 2024-03-29T05:05:54.206 INFO:tasks.workunit.client.0.smithi181.stdout:2: releasing exclusive lock 2024-03-29T05:05:54.208 INFO:tasks.workunit.client.0.smithi181.stdout:2: exclusive lock released 2024-03-29T05:05:54.208 INFO:tasks.workunit.client.0.smithi181.stdout:2: acquiring exclusive lock 2024-03-29T05:05:54.212 INFO:tasks.workunit.client.0.smithi181.stdout:1: exclusive lock acquired 2024-03-29T05:05:54.259 INFO:tasks.workunit.client.0.smithi181.stdout:1: releasing exclusive lock 2024-03-29T05:05:54.261 INFO:tasks.workunit.client.0.smithi181.stdout:1: exclusive lock released 2024-03-29T05:05:54.261 INFO:tasks.workunit.client.0.smithi181.stdout:1: acquiring exclusive lock 2024-03-29T05:05:54.264 INFO:tasks.workunit.client.0.smithi181.stdout:2: exclusive lock acquired 2024-03-29T05:05:54.279 INFO:tasks.workunit.client.0.smithi181.stdout:2: releasing exclusive lock 2024-03-29T05:05:54.281 INFO:tasks.workunit.client.0.smithi181.stdout:2: exclusive lock released 2024-03-29T05:05:54.281 INFO:tasks.workunit.client.0.smithi181.stdout:2: acquiring exclusive lock 2024-03-29T05:05:54.285 INFO:tasks.workunit.client.0.smithi181.stdout:2: exclusive lock acquired 2024-03-29T05:05:54.327 INFO:tasks.workunit.client.0.smithi181.stdout:2: releasing exclusive lock 2024-03-29T05:05:54.329 INFO:tasks.workunit.client.0.smithi181.stdout:2: exclusive lock released 2024-03-29T05:05:54.332 INFO:tasks.workunit.client.0.smithi181.stdout:1: exclusive lock acquired 2024-03-29T05:05:54.374 INFO:tasks.workunit.client.0.smithi181.stdout:1: releasing exclusive lock 2024-03-29T05:05:54.377 INFO:tasks.workunit.client.0.smithi181.stdout:1: exclusive lock released 2024-03-29T05:05:54.377 INFO:tasks.workunit.client.0.smithi181.stdout:1: acquiring exclusive lock 2024-03-29T05:05:54.381 INFO:tasks.workunit.client.0.smithi181.stdout:1: exclusive lock acquired 2024-03-29T05:05:54.388 INFO:tasks.workunit.client.0.smithi181.stdout:1: releasing exclusive lock 2024-03-29T05:05:54.390 INFO:tasks.workunit.client.0.smithi181.stdout:1: exclusive lock released 2024-03-29T05:05:54.390 INFO:tasks.workunit.client.0.smithi181.stdout:1: acquiring exclusive lock 2024-03-29T05:05:54.393 INFO:tasks.workunit.client.0.smithi181.stdout:1: exclusive lock acquired 2024-03-29T05:05:54.410 INFO:tasks.workunit.client.0.smithi181.stdout:1: releasing exclusive lock 2024-03-29T05:05:54.413 INFO:tasks.workunit.client.0.smithi181.stdout:1: exclusive lock released 2024-03-29T05:05:54.413 INFO:tasks.workunit.client.0.smithi181.stdout:1: acquiring exclusive lock 2024-03-29T05:05:54.416 INFO:tasks.workunit.client.0.smithi181.stdout:1: exclusive lock acquired 2024-03-29T05:05:54.419 INFO:tasks.workunit.client.0.smithi181.stdout:1: releasing exclusive lock 2024-03-29T05:05:54.422 INFO:tasks.workunit.client.0.smithi181.stdout:1: exclusive lock released 2024-03-29T05:05:54.435 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ExclusiveLock (380 ms) 2024-03-29T05:05:54.435 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.BreakLock 2024-03-29T05:05:54.450 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:05:54.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:54 smithi181 ceph-mon[101970]: failed to encode map e740 with expected crc 2024-03-29T05:05:54.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:54 smithi181 ceph-mon[101970]: failed to encode map e740 with expected crc 2024-03-29T05:05:54.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:54 smithi181 ceph-mon[101970]: osdmap e741: 8 total, 8 up, 8 in 2024-03-29T05:05:54.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:54 smithi181 ceph-mon[101970]: failed to encode map e741 with expected crc 2024-03-29T05:05:54.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:54 smithi181 ceph-mon[106466]: failed to encode map e740 with expected crc 2024-03-29T05:05:54.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:54 smithi181 ceph-mon[106466]: failed to encode map e740 with expected crc 2024-03-29T05:05:54.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:54 smithi181 ceph-mon[106466]: osdmap e741: 8 total, 8 up, 8 in 2024-03-29T05:05:54.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:54 smithi181 ceph-mon[106466]: failed to encode map e741 with expected crc 2024-03-29T05:05:54.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:54 smithi187 ceph-mon[93536]: failed to encode map e740 with expected crc 2024-03-29T05:05:54.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:54 smithi187 ceph-mon[93536]: failed to encode map e740 with expected crc 2024-03-29T05:05:54.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:54 smithi187 ceph-mon[93536]: osdmap e741: 8 total, 8 up, 8 in 2024-03-29T05:05:54.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:54 smithi187 ceph-mon[93536]: failed to encode map e741 with expected crc 2024-03-29T05:05:55.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:55 smithi181 ceph-mon[101970]: failed to encode map e741 with expected crc 2024-03-29T05:05:55.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:55 smithi181 ceph-mon[101970]: pgmap v819: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 167 KiB/s rd, 140 KiB/s wr, 221 op/s 2024-03-29T05:05:55.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:55 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2279022951"}]: dispatch 2024-03-29T05:05:55.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:55 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2279022951"}]: dispatch 2024-03-29T05:05:55.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:55 smithi181 ceph-mon[106466]: failed to encode map e741 with expected crc 2024-03-29T05:05:55.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:55 smithi181 ceph-mon[106466]: pgmap v819: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 167 KiB/s rd, 140 KiB/s wr, 221 op/s 2024-03-29T05:05:55.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:55 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2279022951"}]: dispatch 2024-03-29T05:05:55.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:55 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2279022951"}]: dispatch 2024-03-29T05:05:55.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:55 smithi187 ceph-mon[93536]: failed to encode map e741 with expected crc 2024-03-29T05:05:55.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:55 smithi187 ceph-mon[93536]: pgmap v819: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 167 KiB/s rd, 140 KiB/s wr, 221 op/s 2024-03-29T05:05:55.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:55 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3057692014' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2279022951"}]: dispatch 2024-03-29T05:05:55.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:55 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2279022951"}]: dispatch 2024-03-29T05:05:56.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:56 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2279022951"}]': finished 2024-03-29T05:05:56.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:56 smithi181 ceph-mon[101970]: failed to encode map e742 with expected crc 2024-03-29T05:05:56.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:56 smithi181 ceph-mon[101970]: osdmap e742: 8 total, 8 up, 8 in 2024-03-29T05:05:56.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:56 smithi181 ceph-mon[101970]: failed to encode map e742 with expected crc 2024-03-29T05:05:56.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:05:56.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:56 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2279022951"}]': finished 2024-03-29T05:05:56.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:56 smithi181 ceph-mon[106466]: failed to encode map e742 with expected crc 2024-03-29T05:05:56.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:56 smithi181 ceph-mon[106466]: osdmap e742: 8 total, 8 up, 8 in 2024-03-29T05:05:56.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:56 smithi181 ceph-mon[106466]: failed to encode map e742 with expected crc 2024-03-29T05:05:56.707 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:05:56.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:56 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.181:0/2279022951"}]': finished 2024-03-29T05:05:56.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:56 smithi187 ceph-mon[93536]: failed to encode map e742 with expected crc 2024-03-29T05:05:56.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:56 smithi187 ceph-mon[93536]: osdmap e742: 8 total, 8 up, 8 in 2024-03-29T05:05:56.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:56 smithi187 ceph-mon[93536]: failed to encode map e742 with expected crc 2024-03-29T05:05:56.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:05:57.706 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:57 smithi181 ceph-mon[101970]: pgmap v821: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 109 KiB/s rd, 124 KiB/s wr, 144 op/s 2024-03-29T05:05:57.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:57 smithi181 ceph-mon[106466]: pgmap v821: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 109 KiB/s rd, 124 KiB/s wr, 144 op/s 2024-03-29T05:05:57.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:57 smithi187 ceph-mon[93536]: pgmap v821: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 109 KiB/s rd, 124 KiB/s wr, 144 op/s 2024-03-29T05:05:58.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:58 smithi181 ceph-mon[106466]: osdmap e743: 8 total, 8 up, 8 in 2024-03-29T05:05:58.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:58 smithi181 ceph-mon[106466]: failed to encode map e743 with expected crc 2024-03-29T05:05:58.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:58 smithi181 ceph-mon[106466]: failed to encode map e743 with expected crc 2024-03-29T05:05:58.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:58 smithi181 ceph-mon[106466]: failed to encode map e743 with expected crc 2024-03-29T05:05:58.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:58 smithi181 ceph-mon[101970]: osdmap e743: 8 total, 8 up, 8 in 2024-03-29T05:05:58.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:58 smithi181 ceph-mon[101970]: failed to encode map e743 with expected crc 2024-03-29T05:05:58.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:58 smithi181 ceph-mon[101970]: failed to encode map e743 with expected crc 2024-03-29T05:05:58.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:58 smithi181 ceph-mon[101970]: failed to encode map e743 with expected crc 2024-03-29T05:05:59.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:58 smithi187 ceph-mon[93536]: osdmap e743: 8 total, 8 up, 8 in 2024-03-29T05:05:59.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:58 smithi187 ceph-mon[93536]: failed to encode map e743 with expected crc 2024-03-29T05:05:59.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:58 smithi187 ceph-mon[93536]: failed to encode map e743 with expected crc 2024-03-29T05:05:59.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:58 smithi187 ceph-mon[93536]: failed to encode map e743 with expected crc 2024-03-29T05:05:59.206 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:05:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:05:58] "GET /metrics HTTP/1.1" 200 74958 "" "Prometheus/2.43.0" 2024-03-29T05:05:59.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:59 smithi181 ceph-mon[106466]: pgmap v823: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 161 KiB/s rd, 55 KiB/s wr, 168 op/s 2024-03-29T05:05:59.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:05:59 smithi181 ceph-mon[106466]: failed to encode map e743 with expected crc 2024-03-29T05:05:59.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:59 smithi181 ceph-mon[101970]: pgmap v823: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 161 KiB/s rd, 55 KiB/s wr, 168 op/s 2024-03-29T05:05:59.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:05:59 smithi181 ceph-mon[101970]: failed to encode map e743 with expected crc 2024-03-29T05:06:00.038 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:59 smithi187 ceph-mon[93536]: pgmap v823: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 161 KiB/s rd, 55 KiB/s wr, 168 op/s 2024-03-29T05:06:00.038 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:05:59 smithi187 ceph-mon[93536]: failed to encode map e743 with expected crc 2024-03-29T05:06:01.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:01 smithi181 ceph-mon[106466]: pgmap v824: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 132 KiB/s rd, 4.8 KiB/s wr, 134 op/s 2024-03-29T05:06:01.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:01 smithi181 ceph-mon[101970]: pgmap v824: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 132 KiB/s rd, 4.8 KiB/s wr, 134 op/s 2024-03-29T05:06:02.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:01 smithi187 ceph-mon[93536]: pgmap v824: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 132 KiB/s rd, 4.8 KiB/s wr, 134 op/s 2024-03-29T05:06:03.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:03 smithi181 ceph-mon[101970]: pgmap v825: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 106 KiB/s rd, 3.9 KiB/s wr, 108 op/s 2024-03-29T05:06:03.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:03 smithi181 ceph-mon[101970]: failed to encode map e744 with expected crc 2024-03-29T05:06:03.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:03 smithi181 ceph-mon[101970]: osdmap e744: 8 total, 8 up, 8 in 2024-03-29T05:06:03.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:03 smithi181 ceph-mon[101970]: failed to encode map e744 with expected crc 2024-03-29T05:06:03.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:03 smithi181 ceph-mon[106466]: pgmap v825: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 106 KiB/s rd, 3.9 KiB/s wr, 108 op/s 2024-03-29T05:06:03.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:03 smithi181 ceph-mon[106466]: failed to encode map e744 with expected crc 2024-03-29T05:06:03.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:03 smithi181 ceph-mon[106466]: osdmap e744: 8 total, 8 up, 8 in 2024-03-29T05:06:03.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:03 smithi181 ceph-mon[106466]: failed to encode map e744 with expected crc 2024-03-29T05:06:04.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:03 smithi187 ceph-mon[93536]: pgmap v825: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 106 KiB/s rd, 3.9 KiB/s wr, 108 op/s 2024-03-29T05:06:04.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:03 smithi187 ceph-mon[93536]: failed to encode map e744 with expected crc 2024-03-29T05:06:04.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:03 smithi187 ceph-mon[93536]: osdmap e744: 8 total, 8 up, 8 in 2024-03-29T05:06:04.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:03 smithi187 ceph-mon[93536]: failed to encode map e744 with expected crc 2024-03-29T05:06:04.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:04 smithi181 ceph-mon[106466]: failed to encode map e744 with expected crc 2024-03-29T05:06:04.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:04 smithi181 ceph-mon[101970]: failed to encode map e744 with expected crc 2024-03-29T05:06:05.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:04 smithi187 ceph-mon[93536]: failed to encode map e744 with expected crc 2024-03-29T05:06:05.444 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.BreakLock (11008 ms) 2024-03-29T05:06:05.444 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.DiscardAfterWrite 2024-03-29T05:06:05.444 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:05.474 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.DiscardAfterWrite (30 ms) 2024-03-29T05:06:05.474 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.DefaultFeatures 2024-03-29T05:06:05.474 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.DefaultFeatures (0 ms) 2024-03-29T05:06:05.474 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurge 2024-03-29T05:06:05.474 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:05.543 INFO:tasks.workunit.client.0.smithi181.stdout:0.00% done 2024-03-29T05:06:05.544 INFO:tasks.workunit.client.0.smithi181.stdout:1.56% done 2024-03-29T05:06:05.544 INFO:tasks.workunit.client.0.smithi181.stdout:3.12% done 2024-03-29T05:06:05.544 INFO:tasks.workunit.client.0.smithi181.stdout:4.69% done 2024-03-29T05:06:05.544 INFO:tasks.workunit.client.0.smithi181.stdout:6.25% done 2024-03-29T05:06:05.544 INFO:tasks.workunit.client.0.smithi181.stdout:7.81% done 2024-03-29T05:06:05.544 INFO:tasks.workunit.client.0.smithi181.stdout:9.38% done 2024-03-29T05:06:05.544 INFO:tasks.workunit.client.0.smithi181.stdout:10.94% done 2024-03-29T05:06:05.544 INFO:tasks.workunit.client.0.smithi181.stdout:12.50% done 2024-03-29T05:06:05.544 INFO:tasks.workunit.client.0.smithi181.stdout:14.06% done 2024-03-29T05:06:05.544 INFO:tasks.workunit.client.0.smithi181.stdout:15.62% done 2024-03-29T05:06:05.544 INFO:tasks.workunit.client.0.smithi181.stdout:17.19% done 2024-03-29T05:06:05.544 INFO:tasks.workunit.client.0.smithi181.stdout:18.75% done 2024-03-29T05:06:05.544 INFO:tasks.workunit.client.0.smithi181.stdout:20.31% done 2024-03-29T05:06:05.544 INFO:tasks.workunit.client.0.smithi181.stdout:21.88% done 2024-03-29T05:06:05.544 INFO:tasks.workunit.client.0.smithi181.stdout:23.44% done 2024-03-29T05:06:05.545 INFO:tasks.workunit.client.0.smithi181.stdout:25.00% done 2024-03-29T05:06:05.545 INFO:tasks.workunit.client.0.smithi181.stdout:26.56% done 2024-03-29T05:06:05.545 INFO:tasks.workunit.client.0.smithi181.stdout:28.12% done 2024-03-29T05:06:05.545 INFO:tasks.workunit.client.0.smithi181.stdout:29.69% done 2024-03-29T05:06:05.545 INFO:tasks.workunit.client.0.smithi181.stdout:31.25% done 2024-03-29T05:06:05.545 INFO:tasks.workunit.client.0.smithi181.stdout:32.81% done 2024-03-29T05:06:05.545 INFO:tasks.workunit.client.0.smithi181.stdout:34.38% done 2024-03-29T05:06:05.545 INFO:tasks.workunit.client.0.smithi181.stdout:35.94% done 2024-03-29T05:06:05.545 INFO:tasks.workunit.client.0.smithi181.stdout:37.50% done 2024-03-29T05:06:05.545 INFO:tasks.workunit.client.0.smithi181.stdout:39.06% done 2024-03-29T05:06:05.545 INFO:tasks.workunit.client.0.smithi181.stdout:40.62% done 2024-03-29T05:06:05.545 INFO:tasks.workunit.client.0.smithi181.stdout:42.19% done 2024-03-29T05:06:05.545 INFO:tasks.workunit.client.0.smithi181.stdout:43.75% done 2024-03-29T05:06:05.545 INFO:tasks.workunit.client.0.smithi181.stdout:45.31% done 2024-03-29T05:06:05.545 INFO:tasks.workunit.client.0.smithi181.stdout:46.88% done 2024-03-29T05:06:05.546 INFO:tasks.workunit.client.0.smithi181.stdout:48.44% done 2024-03-29T05:06:05.546 INFO:tasks.workunit.client.0.smithi181.stdout:50.00% done 2024-03-29T05:06:05.546 INFO:tasks.workunit.client.0.smithi181.stdout:51.56% done 2024-03-29T05:06:05.546 INFO:tasks.workunit.client.0.smithi181.stdout:53.12% done 2024-03-29T05:06:05.546 INFO:tasks.workunit.client.0.smithi181.stdout:54.69% done 2024-03-29T05:06:05.546 INFO:tasks.workunit.client.0.smithi181.stdout:56.25% done 2024-03-29T05:06:05.546 INFO:tasks.workunit.client.0.smithi181.stdout:57.81% done 2024-03-29T05:06:05.546 INFO:tasks.workunit.client.0.smithi181.stdout:59.38% done 2024-03-29T05:06:05.546 INFO:tasks.workunit.client.0.smithi181.stdout:60.94% done 2024-03-29T05:06:05.546 INFO:tasks.workunit.client.0.smithi181.stdout:62.50% done 2024-03-29T05:06:05.546 INFO:tasks.workunit.client.0.smithi181.stdout:64.06% done 2024-03-29T05:06:05.546 INFO:tasks.workunit.client.0.smithi181.stdout:65.62% done 2024-03-29T05:06:05.546 INFO:tasks.workunit.client.0.smithi181.stdout:67.19% done 2024-03-29T05:06:05.546 INFO:tasks.workunit.client.0.smithi181.stdout:68.75% done 2024-03-29T05:06:05.547 INFO:tasks.workunit.client.0.smithi181.stdout:70.31% done 2024-03-29T05:06:05.547 INFO:tasks.workunit.client.0.smithi181.stdout:71.88% done 2024-03-29T05:06:05.547 INFO:tasks.workunit.client.0.smithi181.stdout:73.44% done 2024-03-29T05:06:05.547 INFO:tasks.workunit.client.0.smithi181.stdout:75.00% done 2024-03-29T05:06:05.547 INFO:tasks.workunit.client.0.smithi181.stdout:76.56% done 2024-03-29T05:06:05.547 INFO:tasks.workunit.client.0.smithi181.stdout:78.12% done 2024-03-29T05:06:05.547 INFO:tasks.workunit.client.0.smithi181.stdout:79.69% done 2024-03-29T05:06:05.547 INFO:tasks.workunit.client.0.smithi181.stdout:81.25% done 2024-03-29T05:06:05.547 INFO:tasks.workunit.client.0.smithi181.stdout:82.81% done 2024-03-29T05:06:05.547 INFO:tasks.workunit.client.0.smithi181.stdout:84.38% done 2024-03-29T05:06:05.547 INFO:tasks.workunit.client.0.smithi181.stdout:85.94% done 2024-03-29T05:06:05.547 INFO:tasks.workunit.client.0.smithi181.stdout:87.50% done 2024-03-29T05:06:05.547 INFO:tasks.workunit.client.0.smithi181.stdout:89.06% done 2024-03-29T05:06:05.547 INFO:tasks.workunit.client.0.smithi181.stdout:90.62% done 2024-03-29T05:06:05.547 INFO:tasks.workunit.client.0.smithi181.stdout:92.19% done 2024-03-29T05:06:05.547 INFO:tasks.workunit.client.0.smithi181.stdout:93.75% done 2024-03-29T05:06:05.548 INFO:tasks.workunit.client.0.smithi181.stdout:95.31% done 2024-03-29T05:06:05.548 INFO:tasks.workunit.client.0.smithi181.stdout:96.88% done 2024-03-29T05:06:05.548 INFO:tasks.workunit.client.0.smithi181.stdout:98.44% done 2024-03-29T05:06:05.548 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurge (70 ms) 2024-03-29T05:06:05.548 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay 2024-03-29T05:06:05.548 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:05.601 INFO:tasks.workunit.client.0.smithi181.stdout:0.00% done 2024-03-29T05:06:05.601 INFO:tasks.workunit.client.0.smithi181.stdout:1.56% done 2024-03-29T05:06:05.601 INFO:tasks.workunit.client.0.smithi181.stdout:3.12% done 2024-03-29T05:06:05.601 INFO:tasks.workunit.client.0.smithi181.stdout:4.69% done 2024-03-29T05:06:05.601 INFO:tasks.workunit.client.0.smithi181.stdout:6.25% done 2024-03-29T05:06:05.601 INFO:tasks.workunit.client.0.smithi181.stdout:7.81% done 2024-03-29T05:06:05.601 INFO:tasks.workunit.client.0.smithi181.stdout:9.38% done 2024-03-29T05:06:05.601 INFO:tasks.workunit.client.0.smithi181.stdout:10.94% done 2024-03-29T05:06:05.601 INFO:tasks.workunit.client.0.smithi181.stdout:12.50% done 2024-03-29T05:06:05.601 INFO:tasks.workunit.client.0.smithi181.stdout:14.06% done 2024-03-29T05:06:05.601 INFO:tasks.workunit.client.0.smithi181.stdout:15.62% done 2024-03-29T05:06:05.602 INFO:tasks.workunit.client.0.smithi181.stdout:17.19% done 2024-03-29T05:06:05.602 INFO:tasks.workunit.client.0.smithi181.stdout:18.75% done 2024-03-29T05:06:05.602 INFO:tasks.workunit.client.0.smithi181.stdout:20.31% done 2024-03-29T05:06:05.602 INFO:tasks.workunit.client.0.smithi181.stdout:21.88% done 2024-03-29T05:06:05.602 INFO:tasks.workunit.client.0.smithi181.stdout:23.44% done 2024-03-29T05:06:05.602 INFO:tasks.workunit.client.0.smithi181.stdout:25.00% done 2024-03-29T05:06:05.602 INFO:tasks.workunit.client.0.smithi181.stdout:26.56% done 2024-03-29T05:06:05.602 INFO:tasks.workunit.client.0.smithi181.stdout:28.12% done 2024-03-29T05:06:05.602 INFO:tasks.workunit.client.0.smithi181.stdout:29.69% done 2024-03-29T05:06:05.602 INFO:tasks.workunit.client.0.smithi181.stdout:31.25% done 2024-03-29T05:06:05.602 INFO:tasks.workunit.client.0.smithi181.stdout:32.81% done 2024-03-29T05:06:05.602 INFO:tasks.workunit.client.0.smithi181.stdout:34.38% done 2024-03-29T05:06:05.602 INFO:tasks.workunit.client.0.smithi181.stdout:35.94% done 2024-03-29T05:06:05.602 INFO:tasks.workunit.client.0.smithi181.stdout:37.50% done 2024-03-29T05:06:05.603 INFO:tasks.workunit.client.0.smithi181.stdout:39.06% done 2024-03-29T05:06:05.603 INFO:tasks.workunit.client.0.smithi181.stdout:40.62% done 2024-03-29T05:06:05.603 INFO:tasks.workunit.client.0.smithi181.stdout:42.19% done 2024-03-29T05:06:05.603 INFO:tasks.workunit.client.0.smithi181.stdout:43.75% done 2024-03-29T05:06:05.603 INFO:tasks.workunit.client.0.smithi181.stdout:45.31% done 2024-03-29T05:06:05.603 INFO:tasks.workunit.client.0.smithi181.stdout:46.88% done 2024-03-29T05:06:05.603 INFO:tasks.workunit.client.0.smithi181.stdout:48.44% done 2024-03-29T05:06:05.603 INFO:tasks.workunit.client.0.smithi181.stdout:50.00% done 2024-03-29T05:06:05.603 INFO:tasks.workunit.client.0.smithi181.stdout:51.56% done 2024-03-29T05:06:05.603 INFO:tasks.workunit.client.0.smithi181.stdout:53.12% done 2024-03-29T05:06:05.603 INFO:tasks.workunit.client.0.smithi181.stdout:54.69% done 2024-03-29T05:06:05.603 INFO:tasks.workunit.client.0.smithi181.stdout:56.25% done 2024-03-29T05:06:05.603 INFO:tasks.workunit.client.0.smithi181.stdout:57.81% done 2024-03-29T05:06:05.603 INFO:tasks.workunit.client.0.smithi181.stdout:59.38% done 2024-03-29T05:06:05.603 INFO:tasks.workunit.client.0.smithi181.stdout:60.94% done 2024-03-29T05:06:05.603 INFO:tasks.workunit.client.0.smithi181.stdout:62.50% done 2024-03-29T05:06:05.604 INFO:tasks.workunit.client.0.smithi181.stdout:64.06% done 2024-03-29T05:06:05.604 INFO:tasks.workunit.client.0.smithi181.stdout:65.62% done 2024-03-29T05:06:05.604 INFO:tasks.workunit.client.0.smithi181.stdout:67.19% done 2024-03-29T05:06:05.604 INFO:tasks.workunit.client.0.smithi181.stdout:68.75% done 2024-03-29T05:06:05.604 INFO:tasks.workunit.client.0.smithi181.stdout:70.31% done 2024-03-29T05:06:05.604 INFO:tasks.workunit.client.0.smithi181.stdout:71.88% done 2024-03-29T05:06:05.604 INFO:tasks.workunit.client.0.smithi181.stdout:73.44% done 2024-03-29T05:06:05.604 INFO:tasks.workunit.client.0.smithi181.stdout:75.00% done 2024-03-29T05:06:05.604 INFO:tasks.workunit.client.0.smithi181.stdout:76.56% done 2024-03-29T05:06:05.604 INFO:tasks.workunit.client.0.smithi181.stdout:78.12% done 2024-03-29T05:06:05.604 INFO:tasks.workunit.client.0.smithi181.stdout:79.69% done 2024-03-29T05:06:05.604 INFO:tasks.workunit.client.0.smithi181.stdout:81.25% done 2024-03-29T05:06:05.604 INFO:tasks.workunit.client.0.smithi181.stdout:82.81% done 2024-03-29T05:06:05.604 INFO:tasks.workunit.client.0.smithi181.stdout:84.38% done 2024-03-29T05:06:05.604 INFO:tasks.workunit.client.0.smithi181.stdout:85.94% done 2024-03-29T05:06:05.604 INFO:tasks.workunit.client.0.smithi181.stdout:87.50% done 2024-03-29T05:06:05.605 INFO:tasks.workunit.client.0.smithi181.stdout:89.06% done 2024-03-29T05:06:05.605 INFO:tasks.workunit.client.0.smithi181.stdout:90.62% done 2024-03-29T05:06:05.605 INFO:tasks.workunit.client.0.smithi181.stdout:92.19% done 2024-03-29T05:06:05.605 INFO:tasks.workunit.client.0.smithi181.stdout:93.75% done 2024-03-29T05:06:05.605 INFO:tasks.workunit.client.0.smithi181.stdout:95.31% done 2024-03-29T05:06:05.605 INFO:tasks.workunit.client.0.smithi181.stdout:96.88% done 2024-03-29T05:06:05.605 INFO:tasks.workunit.client.0.smithi181.stdout:98.44% done 2024-03-29T05:06:05.605 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay (57 ms) 2024-03-29T05:06:05.605 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestTrashPurge 2024-03-29T05:06:05.605 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:05.609 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:05.723 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestTrashPurge (123 ms) 2024-03-29T05:06:05.723 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndRestore 2024-03-29T05:06:05.723 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:05.763 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestTrashMoveAndRestore (39 ms) 2024-03-29T05:06:05.763 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestListWatchers 2024-03-29T05:06:05.763 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:05.787 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestListWatchers (24 ms) 2024-03-29T05:06:05.787 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestSetSnapById 2024-03-29T05:06:05.787 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:05.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:05 smithi181 ceph-mon[106466]: pgmap v827: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 106 KiB/s rd, 3.9 KiB/s wr, 108 op/s 2024-03-29T05:06:05.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:05 smithi181 ceph-mon[101970]: pgmap v827: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 106 KiB/s rd, 3.9 KiB/s wr, 108 op/s 2024-03-29T05:06:06.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:05 smithi187 ceph-mon[93536]: pgmap v827: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 106 KiB/s rd, 3.9 KiB/s wr, 108 op/s 2024-03-29T05:06:06.713 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestSetSnapById (926 ms) 2024-03-29T05:06:06.713 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.Namespaces 2024-03-29T05:06:06.741 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.Namespaces (29 ms) 2024-03-29T05:06:06.741 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.NamespacesPP 2024-03-29T05:06:06.768 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:06.773 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:06.844 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.NamespacesPP (102 ms) 2024-03-29T05:06:06.844 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.Migration 2024-03-29T05:06:06.844 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:06.844 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:07.229 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.Migration (386 ms) 2024-03-29T05:06:07.229 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.MigrationPP 2024-03-29T05:06:07.229 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:07.229 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:07.609 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.MigrationPP (379 ms) 2024-03-29T05:06:07.609 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestGetAccessTimestamp 2024-03-29T05:06:07.609 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:07.624 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestGetAccessTimestamp (16 ms) 2024-03-29T05:06:07.625 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.TestGetModifyTimestamp 2024-03-29T05:06:07.625 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:07.639 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.TestGetModifyTimestamp (15 ms) 2024-03-29T05:06:07.639 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ZeroOverlapFlatten 2024-03-29T05:06:07.640 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:07.781 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ZeroOverlapFlatten (141 ms) 2024-03-29T05:06:07.781 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.PoolMetadata 2024-03-29T05:06:07.809 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.PoolMetadata (29 ms) 2024-03-29T05:06:07.809 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.PoolMetadataPP 2024-03-29T05:06:07.829 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.PoolMetadataPP (20 ms) 2024-03-29T05:06:07.829 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.Config 2024-03-29T05:06:07.834 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:07.871 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.Config (42 ms) 2024-03-29T05:06:07.872 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ConfigPP 2024-03-29T05:06:07.876 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:07.912 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ConfigPP (40 ms) 2024-03-29T05:06:07.912 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.PoolStatsPP 2024-03-29T05:06:07.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-mon[106466]: pgmap v828: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 584 B/s rd, 350 B/s wr, 1 op/s 2024-03-29T05:06:07.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-mon[106466]: failed to encode map e745 with expected crc 2024-03-29T05:06:07.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-mon[106466]: osdmap e745: 8 total, 8 up, 8 in 2024-03-29T05:06:07.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-mon[106466]: failed to encode map e745 with expected crc 2024-03-29T05:06:07.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-mon[106466]: failed to encode map e745 with expected crc 2024-03-29T05:06:07.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-mon[106466]: failed to encode map e745 with expected crc 2024-03-29T05:06:07.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-mon[101970]: pgmap v828: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 584 B/s rd, 350 B/s wr, 1 op/s 2024-03-29T05:06:07.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-mon[101970]: failed to encode map e745 with expected crc 2024-03-29T05:06:07.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-mon[101970]: osdmap e745: 8 total, 8 up, 8 in 2024-03-29T05:06:07.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-mon[101970]: failed to encode map e745 with expected crc 2024-03-29T05:06:07.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-mon[101970]: failed to encode map e745 with expected crc 2024-03-29T05:06:07.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-mon[101970]: failed to encode map e745 with expected crc 2024-03-29T05:06:07.957 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:06:07] ENGINE Bus STOPPING 2024-03-29T05:06:07.957 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:06:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T05:06:07.957 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:06:07] ENGINE Bus STOPPED 2024-03-29T05:06:07.957 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:06:07] ENGINE Bus STARTING 2024-03-29T05:06:07.957 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:06:07] ENGINE Serving on http://:::9283 2024-03-29T05:06:07.957 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:06:07 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:06:07] ENGINE Bus STARTED 2024-03-29T05:06:08.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:07 smithi187 ceph-mon[93536]: pgmap v828: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 584 B/s rd, 350 B/s wr, 1 op/s 2024-03-29T05:06:08.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:07 smithi187 ceph-mon[93536]: failed to encode map e745 with expected crc 2024-03-29T05:06:08.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:07 smithi187 ceph-mon[93536]: osdmap e745: 8 total, 8 up, 8 in 2024-03-29T05:06:08.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:07 smithi187 ceph-mon[93536]: failed to encode map e745 with expected crc 2024-03-29T05:06:08.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:07 smithi187 ceph-mon[93536]: failed to encode map e745 with expected crc 2024-03-29T05:06:08.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:07 smithi187 ceph-mon[93536]: failed to encode map e745 with expected crc 2024-03-29T05:06:08.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[106466]: failed to encode map e746 with expected crc 2024-03-29T05:06:08.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[106466]: osdmap e746: 8 total, 8 up, 8 in 2024-03-29T05:06:08.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[106466]: failed to encode map e746 with expected crc 2024-03-29T05:06:08.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[106466]: failed to encode map e746 with expected crc 2024-03-29T05:06:08.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' 2024-03-29T05:06:08.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:06:08.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:06:08.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:06:08.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:06:08.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:06:08.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:06:08.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[101970]: failed to encode map e746 with expected crc 2024-03-29T05:06:08.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[101970]: osdmap e746: 8 total, 8 up, 8 in 2024-03-29T05:06:08.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[101970]: failed to encode map e746 with expected crc 2024-03-29T05:06:08.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[101970]: failed to encode map e746 with expected crc 2024-03-29T05:06:08.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' 2024-03-29T05:06:08.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:06:08.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:06:08.958 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:06:08.958 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:06:08.958 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:06:08.958 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:06:09.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:08 smithi187 ceph-mon[93536]: failed to encode map e746 with expected crc 2024-03-29T05:06:09.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:08 smithi187 ceph-mon[93536]: osdmap e746: 8 total, 8 up, 8 in 2024-03-29T05:06:09.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:08 smithi187 ceph-mon[93536]: failed to encode map e746 with expected crc 2024-03-29T05:06:09.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:08 smithi187 ceph-mon[93536]: failed to encode map e746 with expected crc 2024-03-29T05:06:09.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:08 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' 2024-03-29T05:06:09.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:08 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:06:09.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:08 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:06:09.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:08 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:06:09.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:08 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:06:09.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:08 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:06:09.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:08 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:06:09.456 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:06:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:06:08] "GET /metrics HTTP/1.1" 200 74958 "" "Prometheus/2.43.0" 2024-03-29T05:06:09.817 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:10.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:09 smithi187 ceph-mon[93536]: pgmap v831: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 266 KiB/s rd, 19 KiB/s wr, 352 op/s 2024-03-29T05:06:10.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:09 smithi187 ceph-mon[93536]: pgmap v832: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 277 KiB/s rd, 19 KiB/s wr, 366 op/s 2024-03-29T05:06:10.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:09 smithi187 ceph-mon[93536]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:09 smithi187 ceph-mon[93536]: osdmap e747: 8 total, 8 up, 8 in 2024-03-29T05:06:10.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:09 smithi187 ceph-mon[93536]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:09 smithi187 ceph-mon[93536]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:09 smithi187 ceph-mon[93536]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:09 smithi187 ceph-mon[93536]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:09 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3831994431' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:10.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:09 smithi187 ceph-mon[93536]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[106466]: pgmap v831: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 266 KiB/s rd, 19 KiB/s wr, 352 op/s 2024-03-29T05:06:10.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[106466]: pgmap v832: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 277 KiB/s rd, 19 KiB/s wr, 366 op/s 2024-03-29T05:06:10.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[106466]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[106466]: osdmap e747: 8 total, 8 up, 8 in 2024-03-29T05:06:10.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[106466]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[106466]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[106466]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[106466]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3831994431' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:10.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[106466]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[101970]: pgmap v831: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 266 KiB/s rd, 19 KiB/s wr, 352 op/s 2024-03-29T05:06:10.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[101970]: pgmap v832: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 277 KiB/s rd, 19 KiB/s wr, 366 op/s 2024-03-29T05:06:10.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[101970]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[101970]: osdmap e747: 8 total, 8 up, 8 in 2024-03-29T05:06:10.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[101970]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[101970]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[101970]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[101970]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3831994431' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:10.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:09 smithi181 ceph-mon[101970]: failed to encode map e747 with expected crc 2024-03-29T05:06:10.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:10 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3831994431' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:06:10.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:10 smithi181 ceph-mon[106466]: osdmap e748: 8 total, 8 up, 8 in 2024-03-29T05:06:10.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:10 smithi181 ceph-mon[106466]: failed to encode map e748 with expected crc 2024-03-29T05:06:10.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:10 smithi181 ceph-mon[106466]: failed to encode map e748 with expected crc 2024-03-29T05:06:10.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:10 smithi181 ceph-mon[106466]: failed to encode map e748 with expected crc 2024-03-29T05:06:10.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:10 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3831994431' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:06:10.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:10 smithi181 ceph-mon[101970]: osdmap e748: 8 total, 8 up, 8 in 2024-03-29T05:06:10.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:10 smithi181 ceph-mon[101970]: failed to encode map e748 with expected crc 2024-03-29T05:06:10.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:10 smithi181 ceph-mon[101970]: failed to encode map e748 with expected crc 2024-03-29T05:06:10.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:10 smithi181 ceph-mon[101970]: failed to encode map e748 with expected crc 2024-03-29T05:06:11.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:10 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3831994431' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:06:11.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:10 smithi187 ceph-mon[93536]: osdmap e748: 8 total, 8 up, 8 in 2024-03-29T05:06:11.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:10 smithi187 ceph-mon[93536]: failed to encode map e748 with expected crc 2024-03-29T05:06:11.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:10 smithi187 ceph-mon[93536]: failed to encode map e748 with expected crc 2024-03-29T05:06:11.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:10 smithi187 ceph-mon[93536]: failed to encode map e748 with expected crc 2024-03-29T05:06:11.868 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:11.876 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:11.883 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:12.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:11 smithi187 ceph-mon[93536]: pgmap v835: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 367 KiB/s rd, 27 KiB/s wr, 482 op/s 2024-03-29T05:06:12.215 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:11 smithi187 ceph-mon[93536]: osdmap e749: 8 total, 8 up, 8 in 2024-03-29T05:06:12.216 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:11 smithi187 ceph-mon[93536]: failed to encode map e749 with expected crc 2024-03-29T05:06:12.216 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:06:12.216 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:11 smithi181 ceph-mon[106466]: pgmap v835: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 367 KiB/s rd, 27 KiB/s wr, 482 op/s 2024-03-29T05:06:12.216 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:11 smithi181 ceph-mon[106466]: osdmap e749: 8 total, 8 up, 8 in 2024-03-29T05:06:12.216 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:11 smithi181 ceph-mon[106466]: failed to encode map e749 with expected crc 2024-03-29T05:06:12.216 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:06:12.217 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:11 smithi181 ceph-mon[101970]: pgmap v835: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 367 KiB/s rd, 27 KiB/s wr, 482 op/s 2024-03-29T05:06:12.217 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:11 smithi181 ceph-mon[101970]: osdmap e749: 8 total, 8 up, 8 in 2024-03-29T05:06:12.217 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:11 smithi181 ceph-mon[101970]: failed to encode map e749 with expected crc 2024-03-29T05:06:12.217 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:06:12.907 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.PoolStatsPP (4996 ms) 2024-03-29T05:06:12.907 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ImageSpec 2024-03-29T05:06:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:12 smithi187 ceph-mon[93536]: failed to encode map e750 with expected crc 2024-03-29T05:06:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:12 smithi187 ceph-mon[93536]: osdmap e750: 8 total, 8 up, 8 in 2024-03-29T05:06:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:12 smithi187 ceph-mon[93536]: failed to encode map e750 with expected crc 2024-03-29T05:06:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:12 smithi187 ceph-mon[93536]: failed to encode map e750 with expected crc 2024-03-29T05:06:13.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:12 smithi181 ceph-mon[101970]: failed to encode map e750 with expected crc 2024-03-29T05:06:13.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:12 smithi181 ceph-mon[101970]: osdmap e750: 8 total, 8 up, 8 in 2024-03-29T05:06:13.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:12 smithi181 ceph-mon[101970]: failed to encode map e750 with expected crc 2024-03-29T05:06:13.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:12 smithi181 ceph-mon[101970]: failed to encode map e750 with expected crc 2024-03-29T05:06:13.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:12 smithi181 ceph-mon[106466]: failed to encode map e750 with expected crc 2024-03-29T05:06:13.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:12 smithi181 ceph-mon[106466]: osdmap e750: 8 total, 8 up, 8 in 2024-03-29T05:06:13.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:12 smithi181 ceph-mon[106466]: failed to encode map e750 with expected crc 2024-03-29T05:06:13.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:12 smithi181 ceph-mon[106466]: failed to encode map e750 with expected crc 2024-03-29T05:06:14.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:13 smithi187 ceph-mon[93536]: pgmap v838: 649 pgs: 5 unknown, 644 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 409 KiB/s rd, 28 KiB/s wr, 538 op/s 2024-03-29T05:06:14.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:13 smithi187 ceph-mon[93536]: failed to encode map e751 with expected crc 2024-03-29T05:06:14.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:13 smithi187 ceph-mon[93536]: osdmap e751: 8 total, 8 up, 8 in 2024-03-29T05:06:14.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:13 smithi187 ceph-mon[93536]: failed to encode map e751 with expected crc 2024-03-29T05:06:14.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:13 smithi181 ceph-mon[101970]: pgmap v838: 649 pgs: 5 unknown, 644 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 409 KiB/s rd, 28 KiB/s wr, 538 op/s 2024-03-29T05:06:14.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:13 smithi181 ceph-mon[101970]: failed to encode map e751 with expected crc 2024-03-29T05:06:14.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:13 smithi181 ceph-mon[101970]: osdmap e751: 8 total, 8 up, 8 in 2024-03-29T05:06:14.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:13 smithi181 ceph-mon[101970]: failed to encode map e751 with expected crc 2024-03-29T05:06:14.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:13 smithi181 ceph-mon[106466]: pgmap v838: 649 pgs: 5 unknown, 644 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 409 KiB/s rd, 28 KiB/s wr, 538 op/s 2024-03-29T05:06:14.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:13 smithi181 ceph-mon[106466]: failed to encode map e751 with expected crc 2024-03-29T05:06:14.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:13 smithi181 ceph-mon[106466]: osdmap e751: 8 total, 8 up, 8 in 2024-03-29T05:06:14.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:13 smithi181 ceph-mon[106466]: failed to encode map e751 with expected crc 2024-03-29T05:06:14.859 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:15.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:14 smithi187 ceph-mon[93536]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:14 smithi187 ceph-mon[93536]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:14 smithi187 ceph-mon[93536]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:14 smithi187 ceph-mon[93536]: osdmap e752: 8 total, 8 up, 8 in 2024-03-29T05:06:15.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:14 smithi187 ceph-mon[93536]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:14 smithi187 ceph-mon[93536]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:14 smithi187 ceph-mon[93536]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:14 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/23347972' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:15.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:14 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:15.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[106466]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[106466]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[106466]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[106466]: osdmap e752: 8 total, 8 up, 8 in 2024-03-29T05:06:15.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[106466]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[106466]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[106466]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/23347972' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:15.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:15.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[101970]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[101970]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[101970]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[101970]: osdmap e752: 8 total, 8 up, 8 in 2024-03-29T05:06:15.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[101970]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[101970]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[101970]: failed to encode map e752 with expected crc 2024-03-29T05:06:15.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/23347972' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:15.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:14 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:16.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:15 smithi187 ceph-mon[93536]: pgmap v841: 681 pgs: 32 unknown, 649 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 412 KiB/s rd, 29 KiB/s wr, 542 op/s 2024-03-29T05:06:16.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:15 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:06:16.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:15 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:06:16.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:15 smithi187 ceph-mon[93536]: failed to encode map e753 with expected crc 2024-03-29T05:06:16.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:15 smithi187 ceph-mon[93536]: osdmap e753: 8 total, 8 up, 8 in 2024-03-29T05:06:16.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:15 smithi187 ceph-mon[93536]: failed to encode map e753 with expected crc 2024-03-29T05:06:16.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:15 smithi187 ceph-mon[93536]: failed to encode map e753 with expected crc 2024-03-29T05:06:16.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:15 smithi187 ceph-mon[93536]: failed to encode map e753 with expected crc 2024-03-29T05:06:16.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:15 smithi181 ceph-mon[106466]: pgmap v841: 681 pgs: 32 unknown, 649 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 412 KiB/s rd, 29 KiB/s wr, 542 op/s 2024-03-29T05:06:16.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:15 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:06:16.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:15 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:06:16.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:15 smithi181 ceph-mon[106466]: failed to encode map e753 with expected crc 2024-03-29T05:06:16.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:15 smithi181 ceph-mon[106466]: osdmap e753: 8 total, 8 up, 8 in 2024-03-29T05:06:16.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:15 smithi181 ceph-mon[106466]: failed to encode map e753 with expected crc 2024-03-29T05:06:16.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:15 smithi181 ceph-mon[106466]: failed to encode map e753 with expected crc 2024-03-29T05:06:16.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:15 smithi181 ceph-mon[106466]: failed to encode map e753 with expected crc 2024-03-29T05:06:16.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:15 smithi181 ceph-mon[101970]: pgmap v841: 681 pgs: 32 unknown, 649 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 412 KiB/s rd, 29 KiB/s wr, 542 op/s 2024-03-29T05:06:16.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:15 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:06:16.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:15 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:06:16.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:15 smithi181 ceph-mon[101970]: failed to encode map e753 with expected crc 2024-03-29T05:06:16.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:15 smithi181 ceph-mon[101970]: osdmap e753: 8 total, 8 up, 8 in 2024-03-29T05:06:16.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:15 smithi181 ceph-mon[101970]: failed to encode map e753 with expected crc 2024-03-29T05:06:16.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:15 smithi181 ceph-mon[101970]: failed to encode map e753 with expected crc 2024-03-29T05:06:16.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:15 smithi181 ceph-mon[101970]: failed to encode map e753 with expected crc 2024-03-29T05:06:17.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:16 smithi187 ceph-mon[93536]: osdmap e754: 8 total, 8 up, 8 in 2024-03-29T05:06:17.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:16 smithi187 ceph-mon[93536]: failed to encode map e754 with expected crc 2024-03-29T05:06:17.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:16 smithi187 ceph-mon[93536]: failed to encode map e754 with expected crc 2024-03-29T05:06:17.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:16 smithi187 ceph-mon[93536]: failed to encode map e754 with expected crc 2024-03-29T05:06:17.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:16 smithi187 ceph-mon[93536]: failed to encode map e754 with expected crc 2024-03-29T05:06:17.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:16 smithi181 ceph-mon[106466]: osdmap e754: 8 total, 8 up, 8 in 2024-03-29T05:06:17.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:16 smithi181 ceph-mon[106466]: failed to encode map e754 with expected crc 2024-03-29T05:06:17.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:16 smithi181 ceph-mon[106466]: failed to encode map e754 with expected crc 2024-03-29T05:06:17.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:16 smithi181 ceph-mon[106466]: failed to encode map e754 with expected crc 2024-03-29T05:06:17.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:16 smithi181 ceph-mon[106466]: failed to encode map e754 with expected crc 2024-03-29T05:06:17.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:16 smithi181 ceph-mon[101970]: osdmap e754: 8 total, 8 up, 8 in 2024-03-29T05:06:17.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:16 smithi181 ceph-mon[101970]: failed to encode map e754 with expected crc 2024-03-29T05:06:17.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:16 smithi181 ceph-mon[101970]: failed to encode map e754 with expected crc 2024-03-29T05:06:17.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:16 smithi181 ceph-mon[101970]: failed to encode map e754 with expected crc 2024-03-29T05:06:17.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:16 smithi181 ceph-mon[101970]: failed to encode map e754 with expected crc 2024-03-29T05:06:18.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:17 smithi187 ceph-mon[93536]: pgmap v844: 681 pgs: 32 unknown, 649 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.7 KiB/s rd, 1023 B/s wr, 4 op/s 2024-03-29T05:06:18.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:17 smithi187 ceph-mon[93536]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:06:18.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:17 smithi187 ceph-mon[93536]: failed to encode map e754 with expected crc 2024-03-29T05:06:18.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:17 smithi187 ceph-mon[93536]: osdmap e755: 8 total, 8 up, 8 in 2024-03-29T05:06:18.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:17 smithi187 ceph-mon[93536]: failed to encode map e755 with expected crc 2024-03-29T05:06:18.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:17 smithi181 ceph-mon[106466]: pgmap v844: 681 pgs: 32 unknown, 649 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.7 KiB/s rd, 1023 B/s wr, 4 op/s 2024-03-29T05:06:18.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:17 smithi181 ceph-mon[106466]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:06:18.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:17 smithi181 ceph-mon[106466]: failed to encode map e754 with expected crc 2024-03-29T05:06:18.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:17 smithi181 ceph-mon[106466]: osdmap e755: 8 total, 8 up, 8 in 2024-03-29T05:06:18.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:17 smithi181 ceph-mon[106466]: failed to encode map e755 with expected crc 2024-03-29T05:06:18.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:17 smithi181 ceph-mon[101970]: pgmap v844: 681 pgs: 32 unknown, 649 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.7 KiB/s rd, 1023 B/s wr, 4 op/s 2024-03-29T05:06:18.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:17 smithi181 ceph-mon[101970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:06:18.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:17 smithi181 ceph-mon[101970]: failed to encode map e754 with expected crc 2024-03-29T05:06:18.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:17 smithi181 ceph-mon[101970]: osdmap e755: 8 total, 8 up, 8 in 2024-03-29T05:06:18.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:17 smithi181 ceph-mon[101970]: failed to encode map e755 with expected crc 2024-03-29T05:06:19.021 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ImageSpec (6113 ms) 2024-03-29T05:06:19.021 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.SnapRemoveWithChildMissing 2024-03-29T05:06:19.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:18 smithi187 ceph-mon[93536]: failed to encode map e756 with expected crc 2024-03-29T05:06:19.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:18 smithi187 ceph-mon[93536]: osdmap e756: 8 total, 8 up, 8 in 2024-03-29T05:06:19.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:18 smithi187 ceph-mon[93536]: failed to encode map e756 with expected crc 2024-03-29T05:06:19.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:18 smithi187 ceph-mon[93536]: failed to encode map e756 with expected crc 2024-03-29T05:06:19.206 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:06:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:06:18] "GET /metrics HTTP/1.1" 200 77328 "" "Prometheus/2.43.0" 2024-03-29T05:06:19.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:18 smithi181 ceph-mon[106466]: failed to encode map e756 with expected crc 2024-03-29T05:06:19.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:18 smithi181 ceph-mon[106466]: osdmap e756: 8 total, 8 up, 8 in 2024-03-29T05:06:19.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:18 smithi181 ceph-mon[106466]: failed to encode map e756 with expected crc 2024-03-29T05:06:19.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:18 smithi181 ceph-mon[106466]: failed to encode map e756 with expected crc 2024-03-29T05:06:19.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:18 smithi181 ceph-mon[101970]: failed to encode map e756 with expected crc 2024-03-29T05:06:19.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:18 smithi181 ceph-mon[101970]: osdmap e756: 8 total, 8 up, 8 in 2024-03-29T05:06:19.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:18 smithi181 ceph-mon[101970]: failed to encode map e756 with expected crc 2024-03-29T05:06:19.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:18 smithi181 ceph-mon[101970]: failed to encode map e756 with expected crc 2024-03-29T05:06:20.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:19 smithi187 ceph-mon[93536]: pgmap v847: 681 pgs: 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 4.5 KiB/s wr, 52 op/s 2024-03-29T05:06:20.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:19 smithi187 ceph-mon[93536]: osdmap e757: 8 total, 8 up, 8 in 2024-03-29T05:06:20.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:19 smithi187 ceph-mon[93536]: failed to encode map e757 with expected crc 2024-03-29T05:06:20.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:19 smithi187 ceph-mon[93536]: failed to encode map e757 with expected crc 2024-03-29T05:06:20.137 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:19 smithi187 ceph-mon[93536]: failed to encode map e757 with expected crc 2024-03-29T05:06:20.179 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:19 smithi181 ceph-mon[101970]: pgmap v847: 681 pgs: 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 4.5 KiB/s wr, 52 op/s 2024-03-29T05:06:20.179 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:19 smithi181 ceph-mon[101970]: osdmap e757: 8 total, 8 up, 8 in 2024-03-29T05:06:20.179 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:19 smithi181 ceph-mon[101970]: failed to encode map e757 with expected crc 2024-03-29T05:06:20.179 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:19 smithi181 ceph-mon[101970]: failed to encode map e757 with expected crc 2024-03-29T05:06:20.180 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:19 smithi181 ceph-mon[101970]: failed to encode map e757 with expected crc 2024-03-29T05:06:20.180 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:19 smithi181 ceph-mon[106466]: pgmap v847: 681 pgs: 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 4.5 KiB/s wr, 52 op/s 2024-03-29T05:06:20.180 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:19 smithi181 ceph-mon[106466]: osdmap e757: 8 total, 8 up, 8 in 2024-03-29T05:06:20.180 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:19 smithi181 ceph-mon[106466]: failed to encode map e757 with expected crc 2024-03-29T05:06:20.180 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:19 smithi181 ceph-mon[106466]: failed to encode map e757 with expected crc 2024-03-29T05:06:20.180 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:19 smithi181 ceph-mon[106466]: failed to encode map e757 with expected crc 2024-03-29T05:06:20.904 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:21.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:20 smithi181 ceph-mon[106466]: failed to encode map e758 with expected crc 2024-03-29T05:06:21.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:20 smithi181 ceph-mon[106466]: osdmap e758: 8 total, 8 up, 8 in 2024-03-29T05:06:21.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:20 smithi181 ceph-mon[106466]: failed to encode map e758 with expected crc 2024-03-29T05:06:21.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:20 smithi181 ceph-mon[106466]: failed to encode map e758 with expected crc 2024-03-29T05:06:21.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:20 smithi181 ceph-mon[106466]: failed to encode map e758 with expected crc 2024-03-29T05:06:21.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:20 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3720669730' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:21.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:20 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:21.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:20 smithi181 ceph-mon[106466]: failed to encode map e758 with expected crc 2024-03-29T05:06:21.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:20 smithi181 ceph-mon[101970]: failed to encode map e758 with expected crc 2024-03-29T05:06:21.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:20 smithi181 ceph-mon[101970]: osdmap e758: 8 total, 8 up, 8 in 2024-03-29T05:06:21.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:20 smithi181 ceph-mon[101970]: failed to encode map e758 with expected crc 2024-03-29T05:06:21.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:20 smithi181 ceph-mon[101970]: failed to encode map e758 with expected crc 2024-03-29T05:06:21.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:20 smithi181 ceph-mon[101970]: failed to encode map e758 with expected crc 2024-03-29T05:06:21.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:20 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3720669730' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:21.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:20 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:21.208 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:20 smithi181 ceph-mon[101970]: failed to encode map e758 with expected crc 2024-03-29T05:06:21.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:20 smithi187 ceph-mon[93536]: failed to encode map e758 with expected crc 2024-03-29T05:06:21.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:20 smithi187 ceph-mon[93536]: osdmap e758: 8 total, 8 up, 8 in 2024-03-29T05:06:21.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:20 smithi187 ceph-mon[93536]: failed to encode map e758 with expected crc 2024-03-29T05:06:21.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:20 smithi187 ceph-mon[93536]: failed to encode map e758 with expected crc 2024-03-29T05:06:21.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:20 smithi187 ceph-mon[93536]: failed to encode map e758 with expected crc 2024-03-29T05:06:21.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:20 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3720669730' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:21.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:20 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:21.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:20 smithi187 ceph-mon[93536]: failed to encode map e758 with expected crc 2024-03-29T05:06:22.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:21 smithi181 ceph-mon[106466]: pgmap v850: 713 pgs: 32 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 4.5 KiB/s wr, 52 op/s 2024-03-29T05:06:22.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:21 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:06:22.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:21 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:06:22.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:21 smithi181 ceph-mon[106466]: failed to encode map e759 with expected crc 2024-03-29T05:06:22.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:21 smithi181 ceph-mon[106466]: osdmap e759: 8 total, 8 up, 8 in 2024-03-29T05:06:22.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:21 smithi181 ceph-mon[106466]: failed to encode map e759 with expected crc 2024-03-29T05:06:22.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:21 smithi181 ceph-mon[106466]: failed to encode map e759 with expected crc 2024-03-29T05:06:22.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:21 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:06:22.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:21 smithi181 ceph-mon[101970]: pgmap v850: 713 pgs: 32 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 4.5 KiB/s wr, 52 op/s 2024-03-29T05:06:22.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:21 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:06:22.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:21 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:06:22.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:21 smithi181 ceph-mon[101970]: failed to encode map e759 with expected crc 2024-03-29T05:06:22.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:21 smithi181 ceph-mon[101970]: osdmap e759: 8 total, 8 up, 8 in 2024-03-29T05:06:22.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:21 smithi181 ceph-mon[101970]: failed to encode map e759 with expected crc 2024-03-29T05:06:22.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:21 smithi181 ceph-mon[101970]: failed to encode map e759 with expected crc 2024-03-29T05:06:22.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:21 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:06:22.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:21 smithi187 ceph-mon[93536]: pgmap v850: 713 pgs: 32 unknown, 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 4.5 KiB/s wr, 52 op/s 2024-03-29T05:06:22.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:21 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:06:22.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:21 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:06:22.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:21 smithi187 ceph-mon[93536]: failed to encode map e759 with expected crc 2024-03-29T05:06:22.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:21 smithi187 ceph-mon[93536]: osdmap e759: 8 total, 8 up, 8 in 2024-03-29T05:06:22.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:21 smithi187 ceph-mon[93536]: failed to encode map e759 with expected crc 2024-03-29T05:06:22.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:21 smithi187 ceph-mon[93536]: failed to encode map e759 with expected crc 2024-03-29T05:06:22.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:21 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:06:23.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:22 smithi181 ceph-mon[101970]: failed to encode map e760 with expected crc 2024-03-29T05:06:23.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:22 smithi181 ceph-mon[101970]: osdmap e760: 8 total, 8 up, 8 in 2024-03-29T05:06:23.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:22 smithi181 ceph-mon[101970]: failed to encode map e760 with expected crc 2024-03-29T05:06:23.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:22 smithi181 ceph-mon[101970]: osdmap e761: 8 total, 8 up, 8 in 2024-03-29T05:06:23.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:22 smithi181 ceph-mon[106466]: failed to encode map e760 with expected crc 2024-03-29T05:06:23.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:22 smithi181 ceph-mon[106466]: osdmap e760: 8 total, 8 up, 8 in 2024-03-29T05:06:23.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:22 smithi181 ceph-mon[106466]: failed to encode map e760 with expected crc 2024-03-29T05:06:23.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:22 smithi181 ceph-mon[106466]: osdmap e761: 8 total, 8 up, 8 in 2024-03-29T05:06:23.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:22 smithi187 ceph-mon[93536]: failed to encode map e760 with expected crc 2024-03-29T05:06:23.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:22 smithi187 ceph-mon[93536]: osdmap e760: 8 total, 8 up, 8 in 2024-03-29T05:06:23.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:22 smithi187 ceph-mon[93536]: failed to encode map e760 with expected crc 2024-03-29T05:06:23.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:22 smithi187 ceph-mon[93536]: osdmap e761: 8 total, 8 up, 8 in 2024-03-29T05:06:24.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:23 smithi181 ceph-mon[106466]: pgmap v853: 713 pgs: 4 unknown, 709 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 139 KiB/s rd, 6.5 KiB/s wr, 176 op/s 2024-03-29T05:06:24.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:23 smithi181 ceph-mon[106466]: failed to encode map e761 with expected crc 2024-03-29T05:06:24.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:23 smithi181 ceph-mon[106466]: failed to encode map e761 with expected crc 2024-03-29T05:06:24.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:23 smithi181 ceph-mon[106466]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:06:24.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:23 smithi181 ceph-mon[106466]: osdmap e762: 8 total, 8 up, 8 in 2024-03-29T05:06:24.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:23 smithi181 ceph-mon[101970]: pgmap v853: 713 pgs: 4 unknown, 709 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 139 KiB/s rd, 6.5 KiB/s wr, 176 op/s 2024-03-29T05:06:24.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:23 smithi181 ceph-mon[101970]: failed to encode map e761 with expected crc 2024-03-29T05:06:24.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:23 smithi181 ceph-mon[101970]: failed to encode map e761 with expected crc 2024-03-29T05:06:24.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:23 smithi181 ceph-mon[101970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:06:24.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:23 smithi181 ceph-mon[101970]: osdmap e762: 8 total, 8 up, 8 in 2024-03-29T05:06:24.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:23 smithi187 ceph-mon[93536]: pgmap v853: 713 pgs: 4 unknown, 709 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 139 KiB/s rd, 6.5 KiB/s wr, 176 op/s 2024-03-29T05:06:24.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:23 smithi187 ceph-mon[93536]: failed to encode map e761 with expected crc 2024-03-29T05:06:24.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:23 smithi187 ceph-mon[93536]: failed to encode map e761 with expected crc 2024-03-29T05:06:24.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:23 smithi187 ceph-mon[93536]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:06:24.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:23 smithi187 ceph-mon[93536]: osdmap e762: 8 total, 8 up, 8 in 2024-03-29T05:06:25.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:24 smithi181 ceph-mon[101970]: failed to encode map e762 with expected crc 2024-03-29T05:06:25.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:24 smithi181 ceph-mon[101970]: failed to encode map e762 with expected crc 2024-03-29T05:06:25.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:24 smithi181 ceph-mon[101970]: failed to encode map e762 with expected crc 2024-03-29T05:06:25.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:24 smithi181 ceph-mon[101970]: osdmap e763: 8 total, 8 up, 8 in 2024-03-29T05:06:25.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:24 smithi181 ceph-mon[106466]: failed to encode map e762 with expected crc 2024-03-29T05:06:25.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:24 smithi181 ceph-mon[106466]: failed to encode map e762 with expected crc 2024-03-29T05:06:25.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:24 smithi181 ceph-mon[106466]: failed to encode map e762 with expected crc 2024-03-29T05:06:25.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:24 smithi181 ceph-mon[106466]: osdmap e763: 8 total, 8 up, 8 in 2024-03-29T05:06:25.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:24 smithi187 ceph-mon[93536]: failed to encode map e762 with expected crc 2024-03-29T05:06:25.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:24 smithi187 ceph-mon[93536]: failed to encode map e762 with expected crc 2024-03-29T05:06:25.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:24 smithi187 ceph-mon[93536]: failed to encode map e762 with expected crc 2024-03-29T05:06:25.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:24 smithi187 ceph-mon[93536]: osdmap e763: 8 total, 8 up, 8 in 2024-03-29T05:06:26.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:25 smithi181 ceph-mon[101970]: pgmap v856: 713 pgs: 713 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 140 KiB/s rd, 6.7 KiB/s wr, 177 op/s 2024-03-29T05:06:26.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:25 smithi181 ceph-mon[101970]: failed to encode map e763 with expected crc 2024-03-29T05:06:26.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:25 smithi181 ceph-mon[101970]: failed to encode map e763 with expected crc 2024-03-29T05:06:26.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:25 smithi181 ceph-mon[101970]: osdmap e764: 8 total, 8 up, 8 in 2024-03-29T05:06:26.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:25 smithi181 ceph-mon[106466]: pgmap v856: 713 pgs: 713 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 140 KiB/s rd, 6.7 KiB/s wr, 177 op/s 2024-03-29T05:06:26.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:25 smithi181 ceph-mon[106466]: failed to encode map e763 with expected crc 2024-03-29T05:06:26.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:25 smithi181 ceph-mon[106466]: failed to encode map e763 with expected crc 2024-03-29T05:06:26.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:25 smithi181 ceph-mon[106466]: osdmap e764: 8 total, 8 up, 8 in 2024-03-29T05:06:26.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:25 smithi187 ceph-mon[93536]: pgmap v856: 713 pgs: 713 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 140 KiB/s rd, 6.7 KiB/s wr, 177 op/s 2024-03-29T05:06:26.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:25 smithi187 ceph-mon[93536]: failed to encode map e763 with expected crc 2024-03-29T05:06:26.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:25 smithi187 ceph-mon[93536]: failed to encode map e763 with expected crc 2024-03-29T05:06:26.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:25 smithi187 ceph-mon[93536]: osdmap e764: 8 total, 8 up, 8 in 2024-03-29T05:06:27.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:26 smithi181 ceph-mon[101970]: failed to encode map e764 with expected crc 2024-03-29T05:06:27.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:26 smithi181 ceph-mon[101970]: failed to encode map e764 with expected crc 2024-03-29T05:06:27.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:26 smithi181 ceph-mon[101970]: failed to encode map e764 with expected crc 2024-03-29T05:06:27.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:06:27.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:26 smithi181 ceph-mon[101970]: osdmap e765: 8 total, 8 up, 8 in 2024-03-29T05:06:27.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:26 smithi181 ceph-mon[106466]: failed to encode map e764 with expected crc 2024-03-29T05:06:27.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:26 smithi181 ceph-mon[106466]: failed to encode map e764 with expected crc 2024-03-29T05:06:27.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:26 smithi181 ceph-mon[106466]: failed to encode map e764 with expected crc 2024-03-29T05:06:27.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:06:27.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:26 smithi181 ceph-mon[106466]: osdmap e765: 8 total, 8 up, 8 in 2024-03-29T05:06:27.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:26 smithi187 ceph-mon[93536]: failed to encode map e764 with expected crc 2024-03-29T05:06:27.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:26 smithi187 ceph-mon[93536]: failed to encode map e764 with expected crc 2024-03-29T05:06:27.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:26 smithi187 ceph-mon[93536]: failed to encode map e764 with expected crc 2024-03-29T05:06:27.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:06:27.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:26 smithi187 ceph-mon[93536]: osdmap e765: 8 total, 8 up, 8 in 2024-03-29T05:06:27.707 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 3 2024-03-29T05:06:27.707 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 3 2024-03-29T05:06:27.707 INFO:tasks.workunit.client.0.smithi181.stdout:child: image201 2024-03-29T05:06:27.707 INFO:tasks.workunit.client.0.smithi181.stdout:child: image203 2024-03-29T05:06:27.708 INFO:tasks.workunit.client.0.smithi181.stdout:child: image202 2024-03-29T05:06:27.708 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-15/image201/adc6ef5e24ae 2024-03-29T05:06:27.708 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:06:27.708 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-34/image202/adc6ff79a6a1 2024-03-29T05:06:27.708 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:06:27.708 INFO:tasks.workunit.client.0.smithi181.stdout:found child test-librbd-smithi181-86707-15/image203/adc6683f36 2024-03-29T05:06:27.708 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:06:27.708 INFO:tasks.workunit.client.0.smithi181.stdout:num children is: 0 2024-03-29T05:06:27.708 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 0 2024-03-29T05:06:27.708 INFO:tasks.workunit.client.0.smithi181.stdout:num snaps is: 0 2024-03-29T05:06:27.708 INFO:tasks.workunit.client.0.smithi181.stdout:expected: 0 2024-03-29T05:06:27.708 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.SnapRemoveWithChildMissing (8687 ms) 2024-03-29T05:06:27.708 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.QuiesceWatch 2024-03-29T05:06:28.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:27 smithi187 ceph-mon[93536]: pgmap v859: 681 pgs: 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:06:28.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:27 smithi187 ceph-mon[93536]: failed to encode map e765 with expected crc 2024-03-29T05:06:28.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:27 smithi187 ceph-mon[93536]: failed to encode map e765 with expected crc 2024-03-29T05:06:28.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:27 smithi187 ceph-mon[93536]: failed to encode map e765 with expected crc 2024-03-29T05:06:28.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:27 smithi187 ceph-mon[93536]: osdmap e766: 8 total, 8 up, 8 in 2024-03-29T05:06:28.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:27 smithi181 ceph-mon[101970]: pgmap v859: 681 pgs: 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:06:28.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:27 smithi181 ceph-mon[101970]: failed to encode map e765 with expected crc 2024-03-29T05:06:28.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:27 smithi181 ceph-mon[101970]: failed to encode map e765 with expected crc 2024-03-29T05:06:28.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:27 smithi181 ceph-mon[101970]: failed to encode map e765 with expected crc 2024-03-29T05:06:28.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:27 smithi181 ceph-mon[101970]: osdmap e766: 8 total, 8 up, 8 in 2024-03-29T05:06:28.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:27 smithi181 ceph-mon[106466]: pgmap v859: 681 pgs: 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:06:28.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:27 smithi181 ceph-mon[106466]: failed to encode map e765 with expected crc 2024-03-29T05:06:28.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:27 smithi181 ceph-mon[106466]: failed to encode map e765 with expected crc 2024-03-29T05:06:28.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:27 smithi181 ceph-mon[106466]: failed to encode map e765 with expected crc 2024-03-29T05:06:28.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:27 smithi181 ceph-mon[106466]: osdmap e766: 8 total, 8 up, 8 in 2024-03-29T05:06:29.206 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:06:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:06:28] "GET /metrics HTTP/1.1" 200 79743 "" "Prometheus/2.43.0" 2024-03-29T05:06:29.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:28 smithi181 ceph-mon[101970]: failed to encode map e766 with expected crc 2024-03-29T05:06:29.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:28 smithi181 ceph-mon[101970]: failed to encode map e766 with expected crc 2024-03-29T05:06:29.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:28 smithi181 ceph-mon[101970]: failed to encode map e766 with expected crc 2024-03-29T05:06:29.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:28 smithi181 ceph-mon[101970]: osdmap e767: 8 total, 8 up, 8 in 2024-03-29T05:06:29.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:28 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3508996797' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:29.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:28 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:29.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:28 smithi181 ceph-mon[106466]: failed to encode map e766 with expected crc 2024-03-29T05:06:29.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:28 smithi181 ceph-mon[106466]: failed to encode map e766 with expected crc 2024-03-29T05:06:29.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:28 smithi181 ceph-mon[106466]: failed to encode map e766 with expected crc 2024-03-29T05:06:29.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:28 smithi181 ceph-mon[106466]: osdmap e767: 8 total, 8 up, 8 in 2024-03-29T05:06:29.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:28 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3508996797' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:29.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:28 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:29.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:28 smithi187 ceph-mon[93536]: failed to encode map e766 with expected crc 2024-03-29T05:06:29.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:28 smithi187 ceph-mon[93536]: failed to encode map e766 with expected crc 2024-03-29T05:06:29.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:28 smithi187 ceph-mon[93536]: failed to encode map e766 with expected crc 2024-03-29T05:06:29.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:28 smithi187 ceph-mon[93536]: osdmap e767: 8 total, 8 up, 8 in 2024-03-29T05:06:29.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:28 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3508996797' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:29.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:28 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:06:29.683 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:30.288 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:29 smithi187 ceph-mon[93536]: pgmap v862: 681 pgs: 681 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:06:30.288 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:29 smithi187 ceph-mon[93536]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.288 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:29 smithi187 ceph-mon[93536]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.288 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:29 smithi187 ceph-mon[93536]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.288 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:29 smithi187 ceph-mon[93536]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.288 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:29 smithi187 ceph-mon[93536]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.288 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:29 smithi187 ceph-mon[93536]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.288 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:29 smithi187 ceph-mon[93536]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.288 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:29 smithi187 ceph-mon[93536]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.288 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:29 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:06:30.288 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:29 smithi187 ceph-mon[93536]: osdmap e768: 8 total, 8 up, 8 in 2024-03-29T05:06:30.458 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[106466]: pgmap v862: 681 pgs: 681 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:06:30.458 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[106466]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.458 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[106466]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.458 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[106466]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.458 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[106466]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.458 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[106466]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.458 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[106466]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.458 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[106466]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.458 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[106466]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.458 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:06:30.458 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[106466]: osdmap e768: 8 total, 8 up, 8 in 2024-03-29T05:06:30.459 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[101970]: pgmap v862: 681 pgs: 681 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:06:30.459 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[101970]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.459 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[101970]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.459 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[101970]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.459 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[101970]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.459 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[101970]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.459 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[101970]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.459 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[101970]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.459 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[101970]: failed to encode map e767 with expected crc 2024-03-29T05:06:30.459 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:06:30.459 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:29 smithi181 ceph-mon[101970]: osdmap e768: 8 total, 8 up, 8 in 2024-03-29T05:06:31.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:30 smithi187 ceph-mon[93536]: failed to encode map e768 with expected crc 2024-03-29T05:06:31.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:30 smithi187 ceph-mon[93536]: failed to encode map e768 with expected crc 2024-03-29T05:06:31.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:30 smithi187 ceph-mon[93536]: failed to encode map e768 with expected crc 2024-03-29T05:06:31.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:30 smithi187 ceph-mon[93536]: failed to encode map e768 with expected crc 2024-03-29T05:06:31.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:30 smithi187 ceph-mon[93536]: osdmap e769: 8 total, 8 up, 8 in 2024-03-29T05:06:31.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:30 smithi181 ceph-mon[106466]: failed to encode map e768 with expected crc 2024-03-29T05:06:31.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:30 smithi181 ceph-mon[106466]: failed to encode map e768 with expected crc 2024-03-29T05:06:31.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:30 smithi181 ceph-mon[106466]: failed to encode map e768 with expected crc 2024-03-29T05:06:31.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:30 smithi181 ceph-mon[106466]: failed to encode map e768 with expected crc 2024-03-29T05:06:31.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:30 smithi181 ceph-mon[106466]: osdmap e769: 8 total, 8 up, 8 in 2024-03-29T05:06:31.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:30 smithi181 ceph-mon[101970]: failed to encode map e768 with expected crc 2024-03-29T05:06:31.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:30 smithi181 ceph-mon[101970]: failed to encode map e768 with expected crc 2024-03-29T05:06:31.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:30 smithi181 ceph-mon[101970]: failed to encode map e768 with expected crc 2024-03-29T05:06:31.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:30 smithi181 ceph-mon[101970]: failed to encode map e768 with expected crc 2024-03-29T05:06:31.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:30 smithi181 ceph-mon[101970]: osdmap e769: 8 total, 8 up, 8 in 2024-03-29T05:06:32.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:31 smithi187 ceph-mon[93536]: pgmap v865: 713 pgs: 32 unknown, 681 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 120 KiB/s rd, 9.5 KiB/s wr, 155 op/s 2024-03-29T05:06:32.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:31 smithi187 ceph-mon[93536]: failed to encode map e769 with expected crc 2024-03-29T05:06:32.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:31 smithi187 ceph-mon[93536]: osdmap e770: 8 total, 8 up, 8 in 2024-03-29T05:06:32.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:31 smithi181 ceph-mon[101970]: pgmap v865: 713 pgs: 32 unknown, 681 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 120 KiB/s rd, 9.5 KiB/s wr, 155 op/s 2024-03-29T05:06:32.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:31 smithi181 ceph-mon[101970]: failed to encode map e769 with expected crc 2024-03-29T05:06:32.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:31 smithi181 ceph-mon[101970]: osdmap e770: 8 total, 8 up, 8 in 2024-03-29T05:06:32.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:31 smithi181 ceph-mon[106466]: pgmap v865: 713 pgs: 32 unknown, 681 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 120 KiB/s rd, 9.5 KiB/s wr, 155 op/s 2024-03-29T05:06:32.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:31 smithi181 ceph-mon[106466]: failed to encode map e769 with expected crc 2024-03-29T05:06:32.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:31 smithi181 ceph-mon[106466]: osdmap e770: 8 total, 8 up, 8 in 2024-03-29T05:06:33.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:32 smithi187 ceph-mon[93536]: failed to encode map e770 with expected crc 2024-03-29T05:06:33.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:32 smithi187 ceph-mon[93536]: failed to encode map e770 with expected crc 2024-03-29T05:06:33.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:32 smithi187 ceph-mon[93536]: failed to encode map e770 with expected crc 2024-03-29T05:06:33.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:32 smithi187 ceph-mon[93536]: osdmap e771: 8 total, 8 up, 8 in 2024-03-29T05:06:33.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:32 smithi181 ceph-mon[101970]: failed to encode map e770 with expected crc 2024-03-29T05:06:33.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:32 smithi181 ceph-mon[101970]: failed to encode map e770 with expected crc 2024-03-29T05:06:33.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:32 smithi181 ceph-mon[101970]: failed to encode map e770 with expected crc 2024-03-29T05:06:33.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:32 smithi181 ceph-mon[101970]: osdmap e771: 8 total, 8 up, 8 in 2024-03-29T05:06:33.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:32 smithi181 ceph-mon[106466]: failed to encode map e770 with expected crc 2024-03-29T05:06:33.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:32 smithi181 ceph-mon[106466]: failed to encode map e770 with expected crc 2024-03-29T05:06:33.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:32 smithi181 ceph-mon[106466]: failed to encode map e770 with expected crc 2024-03-29T05:06:33.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:32 smithi181 ceph-mon[106466]: osdmap e771: 8 total, 8 up, 8 in 2024-03-29T05:06:34.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:33 smithi187 ceph-mon[93536]: pgmap v868: 713 pgs: 4 unknown, 709 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 1.2 KiB/s wr, 47 op/s 2024-03-29T05:06:34.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:33 smithi187 ceph-mon[93536]: failed to encode map e771 with expected crc 2024-03-29T05:06:34.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:33 smithi187 ceph-mon[93536]: failed to encode map e771 with expected crc 2024-03-29T05:06:34.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:33 smithi187 ceph-mon[93536]: failed to encode map e771 with expected crc 2024-03-29T05:06:34.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:33 smithi187 ceph-mon[93536]: osdmap e772: 8 total, 8 up, 8 in 2024-03-29T05:06:34.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:33 smithi181 ceph-mon[101970]: pgmap v868: 713 pgs: 4 unknown, 709 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 1.2 KiB/s wr, 47 op/s 2024-03-29T05:06:34.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:33 smithi181 ceph-mon[101970]: failed to encode map e771 with expected crc 2024-03-29T05:06:34.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:33 smithi181 ceph-mon[101970]: failed to encode map e771 with expected crc 2024-03-29T05:06:34.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:33 smithi181 ceph-mon[101970]: failed to encode map e771 with expected crc 2024-03-29T05:06:34.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:33 smithi181 ceph-mon[101970]: osdmap e772: 8 total, 8 up, 8 in 2024-03-29T05:06:34.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:33 smithi181 ceph-mon[106466]: pgmap v868: 713 pgs: 4 unknown, 709 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 1.2 KiB/s wr, 47 op/s 2024-03-29T05:06:34.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:33 smithi181 ceph-mon[106466]: failed to encode map e771 with expected crc 2024-03-29T05:06:34.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:33 smithi181 ceph-mon[106466]: failed to encode map e771 with expected crc 2024-03-29T05:06:34.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:33 smithi181 ceph-mon[106466]: failed to encode map e771 with expected crc 2024-03-29T05:06:34.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:33 smithi181 ceph-mon[106466]: osdmap e772: 8 total, 8 up, 8 in 2024-03-29T05:06:35.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:34 smithi187 ceph-mon[93536]: failed to encode map e772 with expected crc 2024-03-29T05:06:35.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:34 smithi187 ceph-mon[93536]: failed to encode map e772 with expected crc 2024-03-29T05:06:35.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:34 smithi187 ceph-mon[93536]: failed to encode map e772 with expected crc 2024-03-29T05:06:35.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:34 smithi187 ceph-mon[93536]: osdmap e773: 8 total, 8 up, 8 in 2024-03-29T05:06:35.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:34 smithi181 ceph-mon[101970]: failed to encode map e772 with expected crc 2024-03-29T05:06:35.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:34 smithi181 ceph-mon[101970]: failed to encode map e772 with expected crc 2024-03-29T05:06:35.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:34 smithi181 ceph-mon[101970]: failed to encode map e772 with expected crc 2024-03-29T05:06:35.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:34 smithi181 ceph-mon[101970]: osdmap e773: 8 total, 8 up, 8 in 2024-03-29T05:06:35.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:34 smithi181 ceph-mon[106466]: failed to encode map e772 with expected crc 2024-03-29T05:06:35.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:34 smithi181 ceph-mon[106466]: failed to encode map e772 with expected crc 2024-03-29T05:06:35.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:34 smithi181 ceph-mon[106466]: failed to encode map e772 with expected crc 2024-03-29T05:06:35.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:34 smithi181 ceph-mon[106466]: osdmap e773: 8 total, 8 up, 8 in 2024-03-29T05:06:36.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:35 smithi187 ceph-mon[93536]: pgmap v871: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 1.2 KiB/s wr, 47 op/s 2024-03-29T05:06:36.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:35 smithi187 ceph-mon[93536]: failed to encode map e773 with expected crc 2024-03-29T05:06:36.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:35 smithi187 ceph-mon[93536]: failed to encode map e773 with expected crc 2024-03-29T05:06:36.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:35 smithi187 ceph-mon[93536]: osdmap e774: 8 total, 8 up, 8 in 2024-03-29T05:06:36.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:35 smithi181 ceph-mon[106466]: pgmap v871: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 1.2 KiB/s wr, 47 op/s 2024-03-29T05:06:36.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:35 smithi181 ceph-mon[106466]: failed to encode map e773 with expected crc 2024-03-29T05:06:36.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:35 smithi181 ceph-mon[106466]: failed to encode map e773 with expected crc 2024-03-29T05:06:36.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:35 smithi181 ceph-mon[106466]: osdmap e774: 8 total, 8 up, 8 in 2024-03-29T05:06:36.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:35 smithi181 ceph-mon[101970]: pgmap v871: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 1.2 KiB/s wr, 47 op/s 2024-03-29T05:06:36.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:35 smithi181 ceph-mon[101970]: failed to encode map e773 with expected crc 2024-03-29T05:06:36.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:35 smithi181 ceph-mon[101970]: failed to encode map e773 with expected crc 2024-03-29T05:06:36.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:35 smithi181 ceph-mon[101970]: osdmap e774: 8 total, 8 up, 8 in 2024-03-29T05:06:37.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:36 smithi187 ceph-mon[93536]: failed to encode map e774 with expected crc 2024-03-29T05:06:37.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:36 smithi187 ceph-mon[93536]: failed to encode map e774 with expected crc 2024-03-29T05:06:37.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:36 smithi187 ceph-mon[93536]: failed to encode map e774 with expected crc 2024-03-29T05:06:37.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:36 smithi187 ceph-mon[93536]: osdmap e775: 8 total, 8 up, 8 in 2024-03-29T05:06:37.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:36 smithi181 ceph-mon[106466]: failed to encode map e774 with expected crc 2024-03-29T05:06:37.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:36 smithi181 ceph-mon[106466]: failed to encode map e774 with expected crc 2024-03-29T05:06:37.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:36 smithi181 ceph-mon[106466]: failed to encode map e774 with expected crc 2024-03-29T05:06:37.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:36 smithi181 ceph-mon[106466]: osdmap e775: 8 total, 8 up, 8 in 2024-03-29T05:06:37.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:36 smithi181 ceph-mon[101970]: failed to encode map e774 with expected crc 2024-03-29T05:06:37.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:36 smithi181 ceph-mon[101970]: failed to encode map e774 with expected crc 2024-03-29T05:06:37.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:36 smithi181 ceph-mon[101970]: failed to encode map e774 with expected crc 2024-03-29T05:06:37.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:36 smithi181 ceph-mon[101970]: osdmap e775: 8 total, 8 up, 8 in 2024-03-29T05:06:37.771 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.QuiesceWatch (10064 ms) 2024-03-29T05:06:37.829 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.QuiesceWatchPP 2024-03-29T05:06:37.830 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:38.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:37 smithi187 ceph-mon[93536]: pgmap v874: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:06:38.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:37 smithi187 ceph-mon[93536]: failed to encode map e775 with expected crc 2024-03-29T05:06:38.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:37 smithi187 ceph-mon[93536]: failed to encode map e775 with expected crc 2024-03-29T05:06:38.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:37 smithi187 ceph-mon[93536]: failed to encode map e775 with expected crc 2024-03-29T05:06:38.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:37 smithi187 ceph-mon[93536]: failed to encode map e775 with expected crc 2024-03-29T05:06:38.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:37 smithi187 ceph-mon[93536]: osdmap e776: 8 total, 8 up, 8 in 2024-03-29T05:06:38.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:37 smithi181 ceph-mon[101970]: pgmap v874: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:06:38.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:37 smithi181 ceph-mon[101970]: failed to encode map e775 with expected crc 2024-03-29T05:06:38.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:37 smithi181 ceph-mon[101970]: failed to encode map e775 with expected crc 2024-03-29T05:06:38.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:37 smithi181 ceph-mon[101970]: failed to encode map e775 with expected crc 2024-03-29T05:06:38.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:37 smithi181 ceph-mon[101970]: failed to encode map e775 with expected crc 2024-03-29T05:06:38.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:37 smithi181 ceph-mon[101970]: osdmap e776: 8 total, 8 up, 8 in 2024-03-29T05:06:38.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:37 smithi181 ceph-mon[106466]: pgmap v874: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:06:38.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:37 smithi181 ceph-mon[106466]: failed to encode map e775 with expected crc 2024-03-29T05:06:38.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:37 smithi181 ceph-mon[106466]: failed to encode map e775 with expected crc 2024-03-29T05:06:38.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:37 smithi181 ceph-mon[106466]: failed to encode map e775 with expected crc 2024-03-29T05:06:38.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:37 smithi181 ceph-mon[106466]: failed to encode map e775 with expected crc 2024-03-29T05:06:38.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:37 smithi181 ceph-mon[106466]: osdmap e776: 8 total, 8 up, 8 in 2024-03-29T05:06:39.206 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:06:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:06:38] "GET /metrics HTTP/1.1" 200 79743 "" "Prometheus/2.43.0" 2024-03-29T05:06:39.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:38 smithi181 ceph-mon[106466]: failed to encode map e776 with expected crc 2024-03-29T05:06:39.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:38 smithi181 ceph-mon[106466]: failed to encode map e776 with expected crc 2024-03-29T05:06:39.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:38 smithi181 ceph-mon[106466]: failed to encode map e776 with expected crc 2024-03-29T05:06:39.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:38 smithi181 ceph-mon[106466]: osdmap e777: 8 total, 8 up, 8 in 2024-03-29T05:06:39.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:38 smithi181 ceph-mon[101970]: failed to encode map e776 with expected crc 2024-03-29T05:06:39.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:38 smithi181 ceph-mon[101970]: failed to encode map e776 with expected crc 2024-03-29T05:06:39.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:38 smithi181 ceph-mon[101970]: failed to encode map e776 with expected crc 2024-03-29T05:06:39.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:38 smithi181 ceph-mon[101970]: osdmap e777: 8 total, 8 up, 8 in 2024-03-29T05:06:39.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:38 smithi187 ceph-mon[93536]: failed to encode map e776 with expected crc 2024-03-29T05:06:39.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:38 smithi187 ceph-mon[93536]: failed to encode map e776 with expected crc 2024-03-29T05:06:39.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:38 smithi187 ceph-mon[93536]: failed to encode map e776 with expected crc 2024-03-29T05:06:39.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:38 smithi187 ceph-mon[93536]: osdmap e777: 8 total, 8 up, 8 in 2024-03-29T05:06:40.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:39 smithi187 ceph-mon[93536]: pgmap v877: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 6.2 KiB/s wr, 69 op/s 2024-03-29T05:06:40.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:39 smithi187 ceph-mon[93536]: failed to encode map e777 with expected crc 2024-03-29T05:06:40.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:39 smithi187 ceph-mon[93536]: failed to encode map e777 with expected crc 2024-03-29T05:06:40.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:39 smithi187 ceph-mon[93536]: failed to encode map e777 with expected crc 2024-03-29T05:06:40.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:39 smithi187 ceph-mon[93536]: osdmap e778: 8 total, 8 up, 8 in 2024-03-29T05:06:40.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:39 smithi187 ceph-mon[93536]: failed to encode map e778 with expected crc 2024-03-29T05:06:40.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:39 smithi181 ceph-mon[106466]: pgmap v877: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 6.2 KiB/s wr, 69 op/s 2024-03-29T05:06:40.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:39 smithi181 ceph-mon[106466]: failed to encode map e777 with expected crc 2024-03-29T05:06:40.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:39 smithi181 ceph-mon[106466]: failed to encode map e777 with expected crc 2024-03-29T05:06:40.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:39 smithi181 ceph-mon[106466]: failed to encode map e777 with expected crc 2024-03-29T05:06:40.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:39 smithi181 ceph-mon[106466]: osdmap e778: 8 total, 8 up, 8 in 2024-03-29T05:06:40.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:39 smithi181 ceph-mon[106466]: failed to encode map e778 with expected crc 2024-03-29T05:06:40.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:39 smithi181 ceph-mon[101970]: pgmap v877: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 6.2 KiB/s wr, 69 op/s 2024-03-29T05:06:40.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:39 smithi181 ceph-mon[101970]: failed to encode map e777 with expected crc 2024-03-29T05:06:40.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:39 smithi181 ceph-mon[101970]: failed to encode map e777 with expected crc 2024-03-29T05:06:40.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:39 smithi181 ceph-mon[101970]: failed to encode map e777 with expected crc 2024-03-29T05:06:40.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:39 smithi181 ceph-mon[101970]: osdmap e778: 8 total, 8 up, 8 in 2024-03-29T05:06:40.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:39 smithi181 ceph-mon[101970]: failed to encode map e778 with expected crc 2024-03-29T05:06:41.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:41 smithi187 ceph-mon[93536]: failed to encode map e778 with expected crc 2024-03-29T05:06:41.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:41 smithi187 ceph-mon[93536]: failed to encode map e778 with expected crc 2024-03-29T05:06:41.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:41 smithi187 ceph-mon[93536]: osdmap e779: 8 total, 8 up, 8 in 2024-03-29T05:06:41.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:41 smithi187 ceph-mon[93536]: failed to encode map e779 with expected crc 2024-03-29T05:06:41.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:41 smithi181 ceph-mon[106466]: failed to encode map e778 with expected crc 2024-03-29T05:06:41.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:41 smithi181 ceph-mon[106466]: failed to encode map e778 with expected crc 2024-03-29T05:06:41.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:41 smithi181 ceph-mon[106466]: osdmap e779: 8 total, 8 up, 8 in 2024-03-29T05:06:41.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:41 smithi181 ceph-mon[106466]: failed to encode map e779 with expected crc 2024-03-29T05:06:41.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:41 smithi181 ceph-mon[101970]: failed to encode map e778 with expected crc 2024-03-29T05:06:41.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:41 smithi181 ceph-mon[101970]: failed to encode map e778 with expected crc 2024-03-29T05:06:41.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:41 smithi181 ceph-mon[101970]: osdmap e779: 8 total, 8 up, 8 in 2024-03-29T05:06:41.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:41 smithi181 ceph-mon[101970]: failed to encode map e779 with expected crc 2024-03-29T05:06:42.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:42 smithi187 ceph-mon[93536]: pgmap v880: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 6.2 KiB/s wr, 69 op/s 2024-03-29T05:06:42.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:42 smithi187 ceph-mon[93536]: failed to encode map e779 with expected crc 2024-03-29T05:06:42.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:42 smithi187 ceph-mon[93536]: failed to encode map e779 with expected crc 2024-03-29T05:06:42.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:42 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:06:42.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:42 smithi187 ceph-mon[93536]: osdmap e780: 8 total, 8 up, 8 in 2024-03-29T05:06:42.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:42 smithi187 ceph-mon[93536]: failed to encode map e780 with expected crc 2024-03-29T05:06:42.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:42 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:06:42.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:42 smithi181 ceph-mon[106466]: pgmap v880: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 6.2 KiB/s wr, 69 op/s 2024-03-29T05:06:42.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:42 smithi181 ceph-mon[106466]: failed to encode map e779 with expected crc 2024-03-29T05:06:42.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:42 smithi181 ceph-mon[106466]: failed to encode map e779 with expected crc 2024-03-29T05:06:42.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:42 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:06:42.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:42 smithi181 ceph-mon[106466]: osdmap e780: 8 total, 8 up, 8 in 2024-03-29T05:06:42.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:42 smithi181 ceph-mon[106466]: failed to encode map e780 with expected crc 2024-03-29T05:06:42.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:42 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:06:42.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:42 smithi181 ceph-mon[101970]: pgmap v880: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 6.2 KiB/s wr, 69 op/s 2024-03-29T05:06:42.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:42 smithi181 ceph-mon[101970]: failed to encode map e779 with expected crc 2024-03-29T05:06:42.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:42 smithi181 ceph-mon[101970]: failed to encode map e779 with expected crc 2024-03-29T05:06:42.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:42 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:06:42.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:42 smithi181 ceph-mon[101970]: osdmap e780: 8 total, 8 up, 8 in 2024-03-29T05:06:42.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:42 smithi181 ceph-mon[101970]: failed to encode map e780 with expected crc 2024-03-29T05:06:42.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:42 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:06:43.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:43 smithi187 ceph-mon[93536]: failed to encode map e780 with expected crc 2024-03-29T05:06:43.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:43 smithi187 ceph-mon[93536]: failed to encode map e780 with expected crc 2024-03-29T05:06:43.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:43 smithi187 ceph-mon[93536]: failed to encode map e781 with expected crc 2024-03-29T05:06:43.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:43 smithi187 ceph-mon[93536]: osdmap e781: 8 total, 8 up, 8 in 2024-03-29T05:06:43.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:43 smithi181 ceph-mon[106466]: failed to encode map e780 with expected crc 2024-03-29T05:06:43.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:43 smithi181 ceph-mon[106466]: failed to encode map e780 with expected crc 2024-03-29T05:06:43.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:43 smithi181 ceph-mon[106466]: failed to encode map e781 with expected crc 2024-03-29T05:06:43.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:43 smithi181 ceph-mon[106466]: osdmap e781: 8 total, 8 up, 8 in 2024-03-29T05:06:43.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:43 smithi181 ceph-mon[101970]: failed to encode map e780 with expected crc 2024-03-29T05:06:43.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:43 smithi181 ceph-mon[101970]: failed to encode map e780 with expected crc 2024-03-29T05:06:43.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:43 smithi181 ceph-mon[101970]: failed to encode map e781 with expected crc 2024-03-29T05:06:43.457 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:43 smithi181 ceph-mon[101970]: osdmap e781: 8 total, 8 up, 8 in 2024-03-29T05:06:43.796 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.QuiesceWatchPP (6025 ms) 2024-03-29T05:06:43.796 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.QuiesceWatchError 2024-03-29T05:06:43.796 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:44.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:44 smithi187 ceph-mon[93536]: pgmap v883: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 80 KiB/s rd, 5.0 KiB/s wr, 108 op/s 2024-03-29T05:06:44.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:44 smithi187 ceph-mon[93536]: failed to encode map e781 with expected crc 2024-03-29T05:06:44.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:44 smithi187 ceph-mon[93536]: osdmap e782: 8 total, 8 up, 8 in 2024-03-29T05:06:44.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:44 smithi187 ceph-mon[93536]: failed to encode map e782 with expected crc 2024-03-29T05:06:44.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:44 smithi181 ceph-mon[101970]: pgmap v883: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 80 KiB/s rd, 5.0 KiB/s wr, 108 op/s 2024-03-29T05:06:44.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:44 smithi181 ceph-mon[101970]: failed to encode map e781 with expected crc 2024-03-29T05:06:44.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:44 smithi181 ceph-mon[101970]: osdmap e782: 8 total, 8 up, 8 in 2024-03-29T05:06:44.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:44 smithi181 ceph-mon[101970]: failed to encode map e782 with expected crc 2024-03-29T05:06:44.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:44 smithi181 ceph-mon[106466]: pgmap v883: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 80 KiB/s rd, 5.0 KiB/s wr, 108 op/s 2024-03-29T05:06:44.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:44 smithi181 ceph-mon[106466]: failed to encode map e781 with expected crc 2024-03-29T05:06:44.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:44 smithi181 ceph-mon[106466]: osdmap e782: 8 total, 8 up, 8 in 2024-03-29T05:06:44.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:44 smithi181 ceph-mon[106466]: failed to encode map e782 with expected crc 2024-03-29T05:06:45.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:45 smithi187 ceph-mon[93536]: failed to encode map e782 with expected crc 2024-03-29T05:06:45.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:45 smithi187 ceph-mon[93536]: failed to encode map e782 with expected crc 2024-03-29T05:06:45.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:45 smithi187 ceph-mon[93536]: failed to encode map e782 with expected crc 2024-03-29T05:06:45.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:45 smithi187 ceph-mon[93536]: osdmap e783: 8 total, 8 up, 8 in 2024-03-29T05:06:45.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:45 smithi181 ceph-mon[101970]: failed to encode map e782 with expected crc 2024-03-29T05:06:45.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:45 smithi181 ceph-mon[101970]: failed to encode map e782 with expected crc 2024-03-29T05:06:45.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:45 smithi181 ceph-mon[101970]: failed to encode map e782 with expected crc 2024-03-29T05:06:45.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:45 smithi181 ceph-mon[101970]: osdmap e783: 8 total, 8 up, 8 in 2024-03-29T05:06:45.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:45 smithi181 ceph-mon[106466]: failed to encode map e782 with expected crc 2024-03-29T05:06:45.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:45 smithi181 ceph-mon[106466]: failed to encode map e782 with expected crc 2024-03-29T05:06:45.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:45 smithi181 ceph-mon[106466]: failed to encode map e782 with expected crc 2024-03-29T05:06:45.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:45 smithi181 ceph-mon[106466]: osdmap e783: 8 total, 8 up, 8 in 2024-03-29T05:06:46.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:46 smithi187 ceph-mon[93536]: pgmap v886: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 80 KiB/s rd, 5.2 KiB/s wr, 109 op/s 2024-03-29T05:06:46.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:46 smithi187 ceph-mon[93536]: failed to encode map e783 with expected crc 2024-03-29T05:06:46.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:46 smithi187 ceph-mon[93536]: failed to encode map e783 with expected crc 2024-03-29T05:06:46.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:46 smithi187 ceph-mon[93536]: failed to encode map e783 with expected crc 2024-03-29T05:06:46.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:46 smithi187 ceph-mon[93536]: osdmap e784: 8 total, 8 up, 8 in 2024-03-29T05:06:46.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:46 smithi187 ceph-mon[93536]: failed to encode map e784 with expected crc 2024-03-29T05:06:46.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:46 smithi181 ceph-mon[101970]: pgmap v886: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 80 KiB/s rd, 5.2 KiB/s wr, 109 op/s 2024-03-29T05:06:46.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:46 smithi181 ceph-mon[101970]: failed to encode map e783 with expected crc 2024-03-29T05:06:46.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:46 smithi181 ceph-mon[101970]: failed to encode map e783 with expected crc 2024-03-29T05:06:46.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:46 smithi181 ceph-mon[101970]: failed to encode map e783 with expected crc 2024-03-29T05:06:46.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:46 smithi181 ceph-mon[101970]: osdmap e784: 8 total, 8 up, 8 in 2024-03-29T05:06:46.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:46 smithi181 ceph-mon[101970]: failed to encode map e784 with expected crc 2024-03-29T05:06:46.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:46 smithi181 ceph-mon[106466]: pgmap v886: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 80 KiB/s rd, 5.2 KiB/s wr, 109 op/s 2024-03-29T05:06:46.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:46 smithi181 ceph-mon[106466]: failed to encode map e783 with expected crc 2024-03-29T05:06:46.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:46 smithi181 ceph-mon[106466]: failed to encode map e783 with expected crc 2024-03-29T05:06:46.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:46 smithi181 ceph-mon[106466]: failed to encode map e783 with expected crc 2024-03-29T05:06:46.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:46 smithi181 ceph-mon[106466]: osdmap e784: 8 total, 8 up, 8 in 2024-03-29T05:06:46.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:46 smithi181 ceph-mon[106466]: failed to encode map e784 with expected crc 2024-03-29T05:06:47.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:47 smithi187 ceph-mon[93536]: failed to encode map e784 with expected crc 2024-03-29T05:06:47.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:47 smithi187 ceph-mon[93536]: failed to encode map e784 with expected crc 2024-03-29T05:06:47.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:47 smithi187 ceph-mon[93536]: osdmap e785: 8 total, 8 up, 8 in 2024-03-29T05:06:47.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:47 smithi181 ceph-mon[101970]: failed to encode map e784 with expected crc 2024-03-29T05:06:47.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:47 smithi181 ceph-mon[101970]: failed to encode map e784 with expected crc 2024-03-29T05:06:47.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:47 smithi181 ceph-mon[101970]: osdmap e785: 8 total, 8 up, 8 in 2024-03-29T05:06:47.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:47 smithi181 ceph-mon[106466]: failed to encode map e784 with expected crc 2024-03-29T05:06:47.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:47 smithi181 ceph-mon[106466]: failed to encode map e784 with expected crc 2024-03-29T05:06:47.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:47 smithi181 ceph-mon[106466]: osdmap e785: 8 total, 8 up, 8 in 2024-03-29T05:06:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:48 smithi187 ceph-mon[93536]: pgmap v889: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-03-29T05:06:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:48 smithi187 ceph-mon[93536]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:48 smithi187 ceph-mon[93536]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:48 smithi187 ceph-mon[93536]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:48 smithi187 ceph-mon[93536]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:48 smithi187 ceph-mon[93536]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:48 smithi187 ceph-mon[93536]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:48 smithi187 ceph-mon[93536]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:48 smithi187 ceph-mon[93536]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.387 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:48 smithi187 ceph-mon[93536]: osdmap e786: 8 total, 8 up, 8 in 2024-03-29T05:06:48.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[101970]: pgmap v889: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-03-29T05:06:48.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[101970]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[101970]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[101970]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[101970]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[101970]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[101970]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[101970]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[101970]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[101970]: osdmap e786: 8 total, 8 up, 8 in 2024-03-29T05:06:48.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[106466]: pgmap v889: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 255 B/s wr, 0 op/s 2024-03-29T05:06:48.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[106466]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[106466]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[106466]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[106466]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[106466]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[106466]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[106466]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[106466]: failed to encode map e785 with expected crc 2024-03-29T05:06:48.457 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-mon[106466]: osdmap e786: 8 total, 8 up, 8 in 2024-03-29T05:06:49.206 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:06:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:06:48] "GET /metrics HTTP/1.1" 200 82135 "" "Prometheus/2.43.0" 2024-03-29T05:06:49.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:49 smithi181 ceph-mon[106466]: failed to encode map e786 with expected crc 2024-03-29T05:06:49.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:49 smithi181 ceph-mon[106466]: failed to encode map e786 with expected crc 2024-03-29T05:06:49.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:49 smithi181 ceph-mon[106466]: pgmap v892: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 71 KiB/s rd, 4.5 KiB/s wr, 95 op/s 2024-03-29T05:06:49.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:49 smithi181 ceph-mon[106466]: failed to encode map e786 with expected crc 2024-03-29T05:06:49.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:49 smithi181 ceph-mon[106466]: osdmap e787: 8 total, 8 up, 8 in 2024-03-29T05:06:49.207 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:49 smithi181 ceph-mon[106466]: failed to encode map e787 with expected crc 2024-03-29T05:06:49.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:49 smithi181 ceph-mon[101970]: failed to encode map e786 with expected crc 2024-03-29T05:06:49.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:49 smithi181 ceph-mon[101970]: failed to encode map e786 with expected crc 2024-03-29T05:06:49.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:49 smithi181 ceph-mon[101970]: pgmap v892: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 71 KiB/s rd, 4.5 KiB/s wr, 95 op/s 2024-03-29T05:06:49.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:49 smithi181 ceph-mon[101970]: failed to encode map e786 with expected crc 2024-03-29T05:06:49.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:49 smithi181 ceph-mon[101970]: osdmap e787: 8 total, 8 up, 8 in 2024-03-29T05:06:49.207 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:49 smithi181 ceph-mon[101970]: failed to encode map e787 with expected crc 2024-03-29T05:06:49.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:49 smithi187 ceph-mon[93536]: failed to encode map e786 with expected crc 2024-03-29T05:06:49.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:49 smithi187 ceph-mon[93536]: failed to encode map e786 with expected crc 2024-03-29T05:06:49.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:49 smithi187 ceph-mon[93536]: pgmap v892: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 71 KiB/s rd, 4.5 KiB/s wr, 95 op/s 2024-03-29T05:06:49.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:49 smithi187 ceph-mon[93536]: failed to encode map e786 with expected crc 2024-03-29T05:06:49.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:49 smithi187 ceph-mon[93536]: osdmap e787: 8 total, 8 up, 8 in 2024-03-29T05:06:49.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:49 smithi187 ceph-mon[93536]: failed to encode map e787 with expected crc 2024-03-29T05:06:49.782 INFO:tasks.workunit.client.0.smithi181.stdout:10.00% done 2024-03-29T05:06:49.782 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.QuiesceWatchError (5986 ms) 2024-03-29T05:06:49.782 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.QuiesceWatchTimeout 2024-03-29T05:06:49.782 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:06:49.797 INFO:tasks.workunit.client.0.smithi181.stderr:test quiesce is not long enough to time out 2024-03-29T05:06:50.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:50 smithi187 ceph-mon[93536]: failed to encode map e787 with expected crc 2024-03-29T05:06:50.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:50 smithi187 ceph-mon[93536]: failed to encode map e787 with expected crc 2024-03-29T05:06:50.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:50 smithi187 ceph-mon[93536]: osdmap e788: 8 total, 8 up, 8 in 2024-03-29T05:06:50.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:50 smithi187 ceph-mon[93536]: failed to encode map e788 with expected crc 2024-03-29T05:06:50.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:50 smithi181 ceph-mon[106466]: failed to encode map e787 with expected crc 2024-03-29T05:06:50.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:50 smithi181 ceph-mon[106466]: failed to encode map e787 with expected crc 2024-03-29T05:06:50.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:50 smithi181 ceph-mon[106466]: osdmap e788: 8 total, 8 up, 8 in 2024-03-29T05:06:50.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:50 smithi181 ceph-mon[106466]: failed to encode map e788 with expected crc 2024-03-29T05:06:50.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:50 smithi181 ceph-mon[101970]: failed to encode map e787 with expected crc 2024-03-29T05:06:50.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:50 smithi181 ceph-mon[101970]: failed to encode map e787 with expected crc 2024-03-29T05:06:50.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:50 smithi181 ceph-mon[101970]: osdmap e788: 8 total, 8 up, 8 in 2024-03-29T05:06:50.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:50 smithi181 ceph-mon[101970]: failed to encode map e788 with expected crc 2024-03-29T05:06:51.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:51 smithi187 ceph-mon[93536]: failed to encode map e788 with expected crc 2024-03-29T05:06:51.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:51 smithi187 ceph-mon[93536]: failed to encode map e788 with expected crc 2024-03-29T05:06:51.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:51 smithi187 ceph-mon[93536]: pgmap v895: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 71 KiB/s rd, 4.5 KiB/s wr, 95 op/s 2024-03-29T05:06:51.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:51 smithi181 ceph-mon[101970]: failed to encode map e788 with expected crc 2024-03-29T05:06:51.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:51 smithi181 ceph-mon[101970]: failed to encode map e788 with expected crc 2024-03-29T05:06:51.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:51 smithi181 ceph-mon[101970]: pgmap v895: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 71 KiB/s rd, 4.5 KiB/s wr, 95 op/s 2024-03-29T05:06:51.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:51 smithi181 ceph-mon[106466]: failed to encode map e788 with expected crc 2024-03-29T05:06:51.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:51 smithi181 ceph-mon[106466]: failed to encode map e788 with expected crc 2024-03-29T05:06:51.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:51 smithi181 ceph-mon[106466]: pgmap v895: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 71 KiB/s rd, 4.5 KiB/s wr, 95 op/s 2024-03-29T05:06:53.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:53 smithi181 ceph-mon[106466]: pgmap v896: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 5.5 KiB/s wr, 118 op/s 2024-03-29T05:06:53.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:53 smithi181 ceph-mon[106466]: failed to encode map e789 with expected crc 2024-03-29T05:06:53.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:53 smithi181 ceph-mon[106466]: osdmap e789: 8 total, 8 up, 8 in 2024-03-29T05:06:53.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:53 smithi181 ceph-mon[106466]: failed to encode map e789 with expected crc 2024-03-29T05:06:53.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:53 smithi181 ceph-mon[106466]: failed to encode map e789 with expected crc 2024-03-29T05:06:53.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:53 smithi181 ceph-mon[101970]: pgmap v896: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 5.5 KiB/s wr, 118 op/s 2024-03-29T05:06:53.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:53 smithi181 ceph-mon[101970]: failed to encode map e789 with expected crc 2024-03-29T05:06:53.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:53 smithi181 ceph-mon[101970]: osdmap e789: 8 total, 8 up, 8 in 2024-03-29T05:06:53.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:53 smithi181 ceph-mon[101970]: failed to encode map e789 with expected crc 2024-03-29T05:06:53.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:53 smithi181 ceph-mon[101970]: failed to encode map e789 with expected crc 2024-03-29T05:06:54.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:53 smithi187 ceph-mon[93536]: pgmap v896: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 5.5 KiB/s wr, 118 op/s 2024-03-29T05:06:54.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:53 smithi187 ceph-mon[93536]: failed to encode map e789 with expected crc 2024-03-29T05:06:54.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:53 smithi187 ceph-mon[93536]: osdmap e789: 8 total, 8 up, 8 in 2024-03-29T05:06:54.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:53 smithi187 ceph-mon[93536]: failed to encode map e789 with expected crc 2024-03-29T05:06:54.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:53 smithi187 ceph-mon[93536]: failed to encode map e789 with expected crc 2024-03-29T05:06:55.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:55 smithi181 ceph-mon[106466]: pgmap v898: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 3.7 KiB/s wr, 57 op/s 2024-03-29T05:06:55.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:55 smithi181 ceph-mon[101970]: pgmap v898: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 3.7 KiB/s wr, 57 op/s 2024-03-29T05:06:56.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:55 smithi187 ceph-mon[93536]: pgmap v898: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 3.7 KiB/s wr, 57 op/s 2024-03-29T05:06:56.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:06:56.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:06:57.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:06:57.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:57 smithi181 ceph-mon[106466]: pgmap v899: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 2.9 KiB/s wr, 45 op/s 2024-03-29T05:06:57.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:57 smithi181 ceph-mon[101970]: pgmap v899: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 2.9 KiB/s wr, 45 op/s 2024-03-29T05:06:58.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:57 smithi187 ceph-mon[93536]: pgmap v899: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 2.9 KiB/s wr, 45 op/s 2024-03-29T05:06:58.710 INFO:tasks.workunit.client.0.smithi181.stderr:test quiesce is timed out 2024-03-29T05:06:58.717 INFO:tasks.workunit.client.0.smithi181.stderr:waiting for timed out ... 0 2024-03-29T05:06:58.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:58 smithi181 ceph-mon[101970]: failed to encode map e790 with expected crc 2024-03-29T05:06:58.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:58 smithi181 ceph-mon[101970]: osdmap e790: 8 total, 8 up, 8 in 2024-03-29T05:06:58.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:58 smithi181 ceph-mon[101970]: failed to encode map e790 with expected crc 2024-03-29T05:06:58.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:58 smithi181 ceph-mon[101970]: failed to encode map e790 with expected crc 2024-03-29T05:06:58.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:58 smithi181 ceph-mon[106466]: failed to encode map e790 with expected crc 2024-03-29T05:06:58.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:58 smithi181 ceph-mon[106466]: osdmap e790: 8 total, 8 up, 8 in 2024-03-29T05:06:58.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:58 smithi181 ceph-mon[106466]: failed to encode map e790 with expected crc 2024-03-29T05:06:58.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:58 smithi181 ceph-mon[106466]: failed to encode map e790 with expected crc 2024-03-29T05:06:59.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:58 smithi187 ceph-mon[93536]: failed to encode map e790 with expected crc 2024-03-29T05:06:59.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:58 smithi187 ceph-mon[93536]: osdmap e790: 8 total, 8 up, 8 in 2024-03-29T05:06:59.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:58 smithi187 ceph-mon[93536]: failed to encode map e790 with expected crc 2024-03-29T05:06:59.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:58 smithi187 ceph-mon[93536]: failed to encode map e790 with expected crc 2024-03-29T05:06:59.205 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:06:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:06:58] "GET /metrics HTTP/1.1" 200 82158 "" "Prometheus/2.43.0" 2024-03-29T05:06:59.717 INFO:tasks.workunit.client.0.smithi181.stderr:waiting for timed out ... 1 2024-03-29T05:06:59.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:59 smithi181 ceph-mon[101970]: pgmap v901: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 2.7 KiB/s wr, 43 op/s 2024-03-29T05:06:59.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:59 smithi181 ceph-mon[101970]: osdmap e791: 8 total, 8 up, 8 in 2024-03-29T05:06:59.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:59 smithi181 ceph-mon[101970]: failed to encode map e791 with expected crc 2024-03-29T05:06:59.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:59 smithi181 ceph-mon[101970]: failed to encode map e791 with expected crc 2024-03-29T05:06:59.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:06:59 smithi181 ceph-mon[101970]: failed to encode map e791 with expected crc 2024-03-29T05:06:59.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:59 smithi181 ceph-mon[106466]: pgmap v901: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 2.7 KiB/s wr, 43 op/s 2024-03-29T05:06:59.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:59 smithi181 ceph-mon[106466]: osdmap e791: 8 total, 8 up, 8 in 2024-03-29T05:06:59.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:59 smithi181 ceph-mon[106466]: failed to encode map e791 with expected crc 2024-03-29T05:06:59.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:59 smithi181 ceph-mon[106466]: failed to encode map e791 with expected crc 2024-03-29T05:06:59.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:06:59 smithi181 ceph-mon[106466]: failed to encode map e791 with expected crc 2024-03-29T05:07:00.037 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:59 smithi187 ceph-mon[93536]: pgmap v901: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 2.7 KiB/s wr, 43 op/s 2024-03-29T05:07:00.037 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:59 smithi187 ceph-mon[93536]: osdmap e791: 8 total, 8 up, 8 in 2024-03-29T05:07:00.037 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:59 smithi187 ceph-mon[93536]: failed to encode map e791 with expected crc 2024-03-29T05:07:00.037 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:59 smithi187 ceph-mon[93536]: failed to encode map e791 with expected crc 2024-03-29T05:07:00.037 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:06:59 smithi187 ceph-mon[93536]: failed to encode map e791 with expected crc 2024-03-29T05:07:00.717 INFO:tasks.workunit.client.0.smithi181.stderr:waiting for timed out ... 2 2024-03-29T05:07:01.718 INFO:tasks.workunit.client.0.smithi181.stderr:waiting for timed out ... 3 2024-03-29T05:07:01.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:01 smithi181 ceph-mon[101970]: pgmap v903: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.0 KiB/s rd, 665 B/s wr, 2 op/s 2024-03-29T05:07:01.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:01 smithi181 ceph-mon[106466]: pgmap v903: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.0 KiB/s rd, 665 B/s wr, 2 op/s 2024-03-29T05:07:02.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:01 smithi187 ceph-mon[93536]: pgmap v903: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.0 KiB/s rd, 665 B/s wr, 2 op/s 2024-03-29T05:07:02.718 INFO:tasks.workunit.client.0.smithi181.stderr:waiting for timed out ... 4 2024-03-29T05:07:03.718 INFO:tasks.workunit.client.0.smithi181.stderr:waiting for timed out ... 5 2024-03-29T05:07:03.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:03 smithi181 ceph-mon[101970]: pgmap v904: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-29T05:07:03.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:03 smithi181 ceph-mon[106466]: pgmap v904: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-29T05:07:04.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:03 smithi187 ceph-mon[93536]: pgmap v904: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-29T05:07:04.718 INFO:tasks.workunit.client.0.smithi181.stderr:waiting for timed out ... 6 2024-03-29T05:07:05.718 INFO:tasks.workunit.client.0.smithi181.stderr:waiting for timed out ... 7 2024-03-29T05:07:05.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:05 smithi181 ceph-mon[106466]: pgmap v905: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-03-29T05:07:05.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:05 smithi181 ceph-mon[101970]: pgmap v905: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-03-29T05:07:06.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:05 smithi187 ceph-mon[93536]: pgmap v905: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-03-29T05:07:06.718 INFO:tasks.workunit.client.0.smithi181.stderr:waiting for timed out ... 8 2024-03-29T05:07:07.718 INFO:tasks.workunit.client.0.smithi181.stderr:waiting for timed out ... 9 2024-03-29T05:07:07.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:07 smithi181 ceph-mon[101970]: pgmap v906: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 353 B/s wr, 4 op/s 2024-03-29T05:07:07.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:07 smithi181 ceph-mon[106466]: pgmap v906: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 353 B/s wr, 4 op/s 2024-03-29T05:07:08.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:07 smithi187 ceph-mon[93536]: pgmap v906: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 353 B/s wr, 4 op/s 2024-03-29T05:07:08.718 INFO:tasks.workunit.client.0.smithi181.stderr:waiting for timed out ... 10 2024-03-29T05:07:09.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:08 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:07:09.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:08 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:07:09.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:08 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:07:09.206 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:07:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:07:08] "GET /metrics HTTP/1.1" 200 82158 "" "Prometheus/2.43.0" 2024-03-29T05:07:10.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:09 smithi187 ceph-mon[93536]: pgmap v907: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.9 KiB/s rd, 307 B/s wr, 3 op/s 2024-03-29T05:07:10.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:09 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:10.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:09 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:10.232 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:09 smithi181 ceph-mon[106466]: pgmap v907: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.9 KiB/s rd, 307 B/s wr, 3 op/s 2024-03-29T05:07:10.232 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:09 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:10.232 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:09 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:10.232 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:09 smithi181 ceph-mon[101970]: pgmap v907: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.9 KiB/s rd, 307 B/s wr, 3 op/s 2024-03-29T05:07:10.232 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:09 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:10.232 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:09 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:12.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:11 smithi181 ceph-mon[101970]: pgmap v908: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.5 KiB/s rd, 262 B/s wr, 3 op/s 2024-03-29T05:07:12.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:07:12.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:11 smithi181 ceph-mon[106466]: pgmap v908: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.5 KiB/s rd, 262 B/s wr, 3 op/s 2024-03-29T05:07:12.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:07:12.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:11 smithi187 ceph-mon[93536]: pgmap v908: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.5 KiB/s rd, 262 B/s wr, 3 op/s 2024-03-29T05:07:12.386 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:07:13.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:13 smithi187 ceph-mon[93536]: pgmap v909: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-29T05:07:13.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:13.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:13.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.19", "id": [6, 3]}]: dispatch 2024-03-29T05:07:13.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.4", "id": [0, 3]}]: dispatch 2024-03-29T05:07:13.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.12", "id": [0, 3]}]: dispatch 2024-03-29T05:07:13.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.3", "id": [0, 1]}]: dispatch 2024-03-29T05:07:13.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.1a", "id": [7, 1]}]: dispatch 2024-03-29T05:07:13.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]: dispatch 2024-03-29T05:07:13.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]: dispatch 2024-03-29T05:07:13.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.13", "id": [3, 1]}]: dispatch 2024-03-29T05:07:13.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:13.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:13.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[101970]: pgmap v909: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-29T05:07:13.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:13.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:13.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.19", "id": [6, 3]}]: dispatch 2024-03-29T05:07:13.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.4", "id": [0, 3]}]: dispatch 2024-03-29T05:07:13.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.12", "id": [0, 3]}]: dispatch 2024-03-29T05:07:13.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.3", "id": [0, 1]}]: dispatch 2024-03-29T05:07:13.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.1a", "id": [7, 1]}]: dispatch 2024-03-29T05:07:13.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]: dispatch 2024-03-29T05:07:13.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]: dispatch 2024-03-29T05:07:13.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.13", "id": [3, 1]}]: dispatch 2024-03-29T05:07:13.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:13.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:13.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[106466]: pgmap v909: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-29T05:07:13.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:13.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:13.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.19", "id": [6, 3]}]: dispatch 2024-03-29T05:07:13.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.4", "id": [0, 3]}]: dispatch 2024-03-29T05:07:13.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.12", "id": [0, 3]}]: dispatch 2024-03-29T05:07:13.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.3", "id": [0, 1]}]: dispatch 2024-03-29T05:07:13.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.1a", "id": [7, 1]}]: dispatch 2024-03-29T05:07:13.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]: dispatch 2024-03-29T05:07:13.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]: dispatch 2024-03-29T05:07:13.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.13", "id": [3, 1]}]: dispatch 2024-03-29T05:07:13.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:13.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:14.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.19", "id": [6, 3]}]': finished 2024-03-29T05:07:14.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.4", "id": [0, 3]}]': finished 2024-03-29T05:07:14.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.12", "id": [0, 3]}]': finished 2024-03-29T05:07:14.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.3", "id": [0, 1]}]': finished 2024-03-29T05:07:14.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.1a", "id": [7, 1]}]': finished 2024-03-29T05:07:14.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]': finished 2024-03-29T05:07:14.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-03-29T05:07:14.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.13", "id": [3, 1]}]': finished 2024-03-29T05:07:14.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: failed to encode map e792 with expected crc 2024-03-29T05:07:14.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: osdmap e792: 8 total, 8 up, 8 in 2024-03-29T05:07:14.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: failed to encode map e792 with expected crc 2024-03-29T05:07:14.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: failed to encode map e792 with expected crc 2024-03-29T05:07:14.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:07:14.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:07:14.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: pgmap v911: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:14.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: pgmap v912: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:14.887 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:14 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:14.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.19", "id": [6, 3]}]': finished 2024-03-29T05:07:14.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.4", "id": [0, 3]}]': finished 2024-03-29T05:07:14.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.12", "id": [0, 3]}]': finished 2024-03-29T05:07:14.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.3", "id": [0, 1]}]': finished 2024-03-29T05:07:14.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.1a", "id": [7, 1]}]': finished 2024-03-29T05:07:14.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]': finished 2024-03-29T05:07:14.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-03-29T05:07:14.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.13", "id": [3, 1]}]': finished 2024-03-29T05:07:14.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: failed to encode map e792 with expected crc 2024-03-29T05:07:14.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: osdmap e792: 8 total, 8 up, 8 in 2024-03-29T05:07:14.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: failed to encode map e792 with expected crc 2024-03-29T05:07:14.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: failed to encode map e792 with expected crc 2024-03-29T05:07:14.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:07:14.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:07:14.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: pgmap v911: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:14.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: pgmap v912: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:14.957 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:14.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.19", "id": [6, 3]}]': finished 2024-03-29T05:07:14.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.4", "id": [0, 3]}]': finished 2024-03-29T05:07:14.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.12", "id": [0, 3]}]': finished 2024-03-29T05:07:14.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.3", "id": [0, 1]}]': finished 2024-03-29T05:07:14.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.1a", "id": [7, 1]}]': finished 2024-03-29T05:07:14.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.5", "id": [3, 1]}]': finished 2024-03-29T05:07:14.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-03-29T05:07:14.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.13", "id": [3, 1]}]': finished 2024-03-29T05:07:14.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: failed to encode map e792 with expected crc 2024-03-29T05:07:14.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: osdmap e792: 8 total, 8 up, 8 in 2024-03-29T05:07:14.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: failed to encode map e792 with expected crc 2024-03-29T05:07:14.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: failed to encode map e792 with expected crc 2024-03-29T05:07:14.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:07:14.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:07:14.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: pgmap v911: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:14.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: pgmap v912: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:14.958 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:14 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:07:15.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:15 smithi187 ceph-mon[93536]: failed to encode map e792 with expected crc 2024-03-29T05:07:15.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:15 smithi187 ceph-mon[93536]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:15 smithi187 ceph-mon[93536]: osdmap e793: 8 total, 8 up, 8 in 2024-03-29T05:07:15.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:15 smithi187 ceph-mon[93536]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:15 smithi187 ceph-mon[93536]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:15 smithi187 ceph-mon[93536]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:15 smithi187 ceph-mon[93536]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:15 smithi187 ceph-mon[93536]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.886 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:15 smithi187 ceph-mon[93536]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[101970]: failed to encode map e792 with expected crc 2024-03-29T05:07:15.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[101970]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[101970]: osdmap e793: 8 total, 8 up, 8 in 2024-03-29T05:07:15.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[101970]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[101970]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[101970]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[101970]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[101970]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[101970]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[106466]: failed to encode map e792 with expected crc 2024-03-29T05:07:15.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[106466]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[106466]: osdmap e793: 8 total, 8 up, 8 in 2024-03-29T05:07:15.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[106466]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[106466]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[106466]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[106466]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[106466]: failed to encode map e793 with expected crc 2024-03-29T05:07:15.957 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:15 smithi181 ceph-mon[106466]: failed to encode map e793 with expected crc 2024-03-29T05:07:16.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:16 smithi187 ceph-mon[93536]: pgmap v914: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:16.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:16 smithi181 ceph-mon[106466]: pgmap v914: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:16.956 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:16 smithi181 ceph-mon[101970]: pgmap v914: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:19.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:18 smithi187 ceph-mon[93536]: pgmap v915: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6 B/s, 1 keys/s, 0 objects/s recovering 2024-03-29T05:07:19.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:18 smithi181 ceph-mon[106466]: pgmap v915: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6 B/s, 1 keys/s, 0 objects/s recovering 2024-03-29T05:07:19.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:18 smithi181 ceph-mon[101970]: pgmap v915: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6 B/s, 1 keys/s, 0 objects/s recovering 2024-03-29T05:07:19.205 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:07:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:07:18] "GET /metrics HTTP/1.1" 200 82151 "" "Prometheus/2.43.0" 2024-03-29T05:07:19.718 INFO:tasks.workunit.client.0.smithi181.stderr:test retry succeeds 2024-03-29T05:07:21.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:20 smithi187 ceph-mon[93536]: pgmap v916: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5 B/s, 1 keys/s, 0 objects/s recovering 2024-03-29T05:07:21.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:20 smithi187 ceph-mon[93536]: failed to encode map e794 with expected crc 2024-03-29T05:07:21.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:20 smithi187 ceph-mon[93536]: osdmap e794: 8 total, 8 up, 8 in 2024-03-29T05:07:21.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:20 smithi187 ceph-mon[93536]: failed to encode map e794 with expected crc 2024-03-29T05:07:21.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:20 smithi187 ceph-mon[93536]: failed to encode map e794 with expected crc 2024-03-29T05:07:21.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:20 smithi187 ceph-mon[93536]: failed to encode map e794 with expected crc 2024-03-29T05:07:21.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:20 smithi181 ceph-mon[101970]: pgmap v916: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5 B/s, 1 keys/s, 0 objects/s recovering 2024-03-29T05:07:21.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:20 smithi181 ceph-mon[101970]: failed to encode map e794 with expected crc 2024-03-29T05:07:21.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:20 smithi181 ceph-mon[101970]: osdmap e794: 8 total, 8 up, 8 in 2024-03-29T05:07:21.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:20 smithi181 ceph-mon[101970]: failed to encode map e794 with expected crc 2024-03-29T05:07:21.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:20 smithi181 ceph-mon[101970]: failed to encode map e794 with expected crc 2024-03-29T05:07:21.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:20 smithi181 ceph-mon[101970]: failed to encode map e794 with expected crc 2024-03-29T05:07:21.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:20 smithi181 ceph-mon[106466]: pgmap v916: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5 B/s, 1 keys/s, 0 objects/s recovering 2024-03-29T05:07:21.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:20 smithi181 ceph-mon[106466]: failed to encode map e794 with expected crc 2024-03-29T05:07:21.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:20 smithi181 ceph-mon[106466]: osdmap e794: 8 total, 8 up, 8 in 2024-03-29T05:07:21.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:20 smithi181 ceph-mon[106466]: failed to encode map e794 with expected crc 2024-03-29T05:07:21.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:20 smithi181 ceph-mon[106466]: failed to encode map e794 with expected crc 2024-03-29T05:07:21.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:20 smithi181 ceph-mon[106466]: failed to encode map e794 with expected crc 2024-03-29T05:07:21.866 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.QuiesceWatchTimeout (32085 ms) 2024-03-29T05:07:21.867 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.WriteZeroes 2024-03-29T05:07:21.867 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:07:21.928 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.WriteZeroes (61 ms) 2024-03-29T05:07:21.928 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.WriteZeroesThickProvision 2024-03-29T05:07:21.928 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:07:21.998 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.WriteZeroesThickProvision (70 ms) 2024-03-29T05:07:21.998 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] TestLibRBD.ConcurentOperations 2024-03-29T05:07:21.998 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:07:22.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:21 smithi187 ceph-mon[93536]: failed to encode map e795 with expected crc 2024-03-29T05:07:22.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:21 smithi187 ceph-mon[93536]: osdmap e795: 8 total, 8 up, 8 in 2024-03-29T05:07:22.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:21 smithi187 ceph-mon[93536]: failed to encode map e795 with expected crc 2024-03-29T05:07:22.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:21 smithi187 ceph-mon[93536]: failed to encode map e795 with expected crc 2024-03-29T05:07:22.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:21 smithi181 ceph-mon[101970]: failed to encode map e795 with expected crc 2024-03-29T05:07:22.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:21 smithi181 ceph-mon[101970]: osdmap e795: 8 total, 8 up, 8 in 2024-03-29T05:07:22.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:21 smithi181 ceph-mon[101970]: failed to encode map e795 with expected crc 2024-03-29T05:07:22.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:21 smithi181 ceph-mon[101970]: failed to encode map e795 with expected crc 2024-03-29T05:07:22.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:21 smithi181 ceph-mon[106466]: failed to encode map e795 with expected crc 2024-03-29T05:07:22.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:21 smithi181 ceph-mon[106466]: osdmap e795: 8 total, 8 up, 8 in 2024-03-29T05:07:22.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:21 smithi181 ceph-mon[106466]: failed to encode map e795 with expected crc 2024-03-29T05:07:22.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:21 smithi181 ceph-mon[106466]: failed to encode map e795 with expected crc 2024-03-29T05:07:23.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:22 smithi187 ceph-mon[93536]: pgmap v919: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.8 KiB/s rd, 1.3 KiB/s wr, 8 op/s; 4 B/s, 1 keys/s, 0 objects/s recovering 2024-03-29T05:07:23.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:22 smithi187 ceph-mon[93536]: failed to encode map e796 with expected crc 2024-03-29T05:07:23.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:22 smithi187 ceph-mon[93536]: osdmap e796: 8 total, 8 up, 8 in 2024-03-29T05:07:23.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:22 smithi187 ceph-mon[93536]: failed to encode map e796 with expected crc 2024-03-29T05:07:23.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:22 smithi187 ceph-mon[93536]: failed to encode map e796 with expected crc 2024-03-29T05:07:23.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:22 smithi187 ceph-mon[93536]: failed to encode map e796 with expected crc 2024-03-29T05:07:23.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:22 smithi181 ceph-mon[101970]: pgmap v919: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.8 KiB/s rd, 1.3 KiB/s wr, 8 op/s; 4 B/s, 1 keys/s, 0 objects/s recovering 2024-03-29T05:07:23.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:22 smithi181 ceph-mon[101970]: failed to encode map e796 with expected crc 2024-03-29T05:07:23.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:22 smithi181 ceph-mon[101970]: osdmap e796: 8 total, 8 up, 8 in 2024-03-29T05:07:23.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:22 smithi181 ceph-mon[101970]: failed to encode map e796 with expected crc 2024-03-29T05:07:23.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:22 smithi181 ceph-mon[101970]: failed to encode map e796 with expected crc 2024-03-29T05:07:23.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:22 smithi181 ceph-mon[101970]: failed to encode map e796 with expected crc 2024-03-29T05:07:23.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:22 smithi181 ceph-mon[106466]: pgmap v919: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.8 KiB/s rd, 1.3 KiB/s wr, 8 op/s; 4 B/s, 1 keys/s, 0 objects/s recovering 2024-03-29T05:07:23.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:22 smithi181 ceph-mon[106466]: failed to encode map e796 with expected crc 2024-03-29T05:07:23.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:22 smithi181 ceph-mon[106466]: osdmap e796: 8 total, 8 up, 8 in 2024-03-29T05:07:23.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:22 smithi181 ceph-mon[106466]: failed to encode map e796 with expected crc 2024-03-29T05:07:23.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:22 smithi181 ceph-mon[106466]: failed to encode map e796 with expected crc 2024-03-29T05:07:23.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:22 smithi181 ceph-mon[106466]: failed to encode map e796 with expected crc 2024-03-29T05:07:24.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:23 smithi187 ceph-mon[93536]: failed to encode map e797 with expected crc 2024-03-29T05:07:24.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:23 smithi187 ceph-mon[93536]: osdmap e797: 8 total, 8 up, 8 in 2024-03-29T05:07:24.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:23 smithi187 ceph-mon[93536]: failed to encode map e797 with expected crc 2024-03-29T05:07:24.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:23 smithi187 ceph-mon[93536]: failed to encode map e797 with expected crc 2024-03-29T05:07:24.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:23 smithi187 ceph-mon[93536]: failed to encode map e797 with expected crc 2024-03-29T05:07:24.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:23 smithi181 ceph-mon[101970]: failed to encode map e797 with expected crc 2024-03-29T05:07:24.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:23 smithi181 ceph-mon[101970]: osdmap e797: 8 total, 8 up, 8 in 2024-03-29T05:07:24.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:23 smithi181 ceph-mon[101970]: failed to encode map e797 with expected crc 2024-03-29T05:07:24.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:23 smithi181 ceph-mon[101970]: failed to encode map e797 with expected crc 2024-03-29T05:07:24.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:23 smithi181 ceph-mon[101970]: failed to encode map e797 with expected crc 2024-03-29T05:07:24.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:23 smithi181 ceph-mon[106466]: failed to encode map e797 with expected crc 2024-03-29T05:07:24.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:23 smithi181 ceph-mon[106466]: osdmap e797: 8 total, 8 up, 8 in 2024-03-29T05:07:24.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:23 smithi181 ceph-mon[106466]: failed to encode map e797 with expected crc 2024-03-29T05:07:24.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:23 smithi181 ceph-mon[106466]: failed to encode map e797 with expected crc 2024-03-29T05:07:24.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:23 smithi181 ceph-mon[106466]: failed to encode map e797 with expected crc 2024-03-29T05:07:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:24 smithi187 ceph-mon[93536]: pgmap v922: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 7.2 KiB/s wr, 67 op/s 2024-03-29T05:07:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:24 smithi187 ceph-mon[93536]: failed to encode map e798 with expected crc 2024-03-29T05:07:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:24 smithi187 ceph-mon[93536]: osdmap e798: 8 total, 8 up, 8 in 2024-03-29T05:07:25.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:24 smithi187 ceph-mon[93536]: failed to encode map e798 with expected crc 2024-03-29T05:07:25.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:24 smithi187 ceph-mon[93536]: failed to encode map e798 with expected crc 2024-03-29T05:07:25.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:24 smithi181 ceph-mon[101970]: pgmap v922: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 7.2 KiB/s wr, 67 op/s 2024-03-29T05:07:25.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:24 smithi181 ceph-mon[101970]: failed to encode map e798 with expected crc 2024-03-29T05:07:25.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:24 smithi181 ceph-mon[101970]: osdmap e798: 8 total, 8 up, 8 in 2024-03-29T05:07:25.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:24 smithi181 ceph-mon[101970]: failed to encode map e798 with expected crc 2024-03-29T05:07:25.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:24 smithi181 ceph-mon[101970]: failed to encode map e798 with expected crc 2024-03-29T05:07:25.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:24 smithi181 ceph-mon[106466]: pgmap v922: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 7.2 KiB/s wr, 67 op/s 2024-03-29T05:07:25.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:24 smithi181 ceph-mon[106466]: failed to encode map e798 with expected crc 2024-03-29T05:07:25.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:24 smithi181 ceph-mon[106466]: osdmap e798: 8 total, 8 up, 8 in 2024-03-29T05:07:25.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:24 smithi181 ceph-mon[106466]: failed to encode map e798 with expected crc 2024-03-29T05:07:25.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:24 smithi181 ceph-mon[106466]: failed to encode map e798 with expected crc 2024-03-29T05:07:26.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:25 smithi187 ceph-mon[93536]: failed to encode map e799 with expected crc 2024-03-29T05:07:26.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:25 smithi187 ceph-mon[93536]: osdmap e799: 8 total, 8 up, 8 in 2024-03-29T05:07:26.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:25 smithi187 ceph-mon[93536]: failed to encode map e799 with expected crc 2024-03-29T05:07:26.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:25 smithi187 ceph-mon[93536]: failed to encode map e799 with expected crc 2024-03-29T05:07:26.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:25 smithi181 ceph-mon[101970]: failed to encode map e799 with expected crc 2024-03-29T05:07:26.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:25 smithi181 ceph-mon[101970]: osdmap e799: 8 total, 8 up, 8 in 2024-03-29T05:07:26.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:25 smithi181 ceph-mon[101970]: failed to encode map e799 with expected crc 2024-03-29T05:07:26.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:25 smithi181 ceph-mon[101970]: failed to encode map e799 with expected crc 2024-03-29T05:07:26.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:25 smithi181 ceph-mon[106466]: failed to encode map e799 with expected crc 2024-03-29T05:07:26.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:25 smithi181 ceph-mon[106466]: osdmap e799: 8 total, 8 up, 8 in 2024-03-29T05:07:26.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:25 smithi181 ceph-mon[106466]: failed to encode map e799 with expected crc 2024-03-29T05:07:26.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:25 smithi181 ceph-mon[106466]: failed to encode map e799 with expected crc 2024-03-29T05:07:27.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:26 smithi187 ceph-mon[93536]: pgmap v925: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 5.0 KiB/s wr, 47 op/s 2024-03-29T05:07:27.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:26 smithi187 ceph-mon[93536]: failed to encode map e800 with expected crc 2024-03-29T05:07:27.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:26 smithi187 ceph-mon[93536]: osdmap e800: 8 total, 8 up, 8 in 2024-03-29T05:07:27.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:26 smithi187 ceph-mon[93536]: failed to encode map e800 with expected crc 2024-03-29T05:07:27.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:26 smithi187 ceph-mon[93536]: failed to encode map e800 with expected crc 2024-03-29T05:07:27.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:07:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:26 smithi181 ceph-mon[101970]: pgmap v925: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 5.0 KiB/s wr, 47 op/s 2024-03-29T05:07:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:26 smithi181 ceph-mon[101970]: failed to encode map e800 with expected crc 2024-03-29T05:07:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:26 smithi181 ceph-mon[101970]: osdmap e800: 8 total, 8 up, 8 in 2024-03-29T05:07:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:26 smithi181 ceph-mon[101970]: failed to encode map e800 with expected crc 2024-03-29T05:07:27.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:26 smithi181 ceph-mon[101970]: failed to encode map e800 with expected crc 2024-03-29T05:07:27.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:07:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:26 smithi181 ceph-mon[106466]: pgmap v925: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 5.0 KiB/s wr, 47 op/s 2024-03-29T05:07:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:26 smithi181 ceph-mon[106466]: failed to encode map e800 with expected crc 2024-03-29T05:07:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:26 smithi181 ceph-mon[106466]: osdmap e800: 8 total, 8 up, 8 in 2024-03-29T05:07:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:26 smithi181 ceph-mon[106466]: failed to encode map e800 with expected crc 2024-03-29T05:07:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:26 smithi181 ceph-mon[106466]: failed to encode map e800 with expected crc 2024-03-29T05:07:27.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:07:28.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:27 smithi187 ceph-mon[93536]: failed to encode map e801 with expected crc 2024-03-29T05:07:28.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:27 smithi187 ceph-mon[93536]: osdmap e801: 8 total, 8 up, 8 in 2024-03-29T05:07:28.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:27 smithi187 ceph-mon[93536]: failed to encode map e801 with expected crc 2024-03-29T05:07:28.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:27 smithi187 ceph-mon[93536]: failed to encode map e801 with expected crc 2024-03-29T05:07:28.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:27 smithi187 ceph-mon[93536]: osdmap e802: 8 total, 8 up, 8 in 2024-03-29T05:07:28.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:27 smithi181 ceph-mon[101970]: failed to encode map e801 with expected crc 2024-03-29T05:07:28.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:27 smithi181 ceph-mon[101970]: osdmap e801: 8 total, 8 up, 8 in 2024-03-29T05:07:28.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:27 smithi181 ceph-mon[101970]: failed to encode map e801 with expected crc 2024-03-29T05:07:28.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:27 smithi181 ceph-mon[101970]: failed to encode map e801 with expected crc 2024-03-29T05:07:28.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:27 smithi181 ceph-mon[101970]: osdmap e802: 8 total, 8 up, 8 in 2024-03-29T05:07:28.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:27 smithi181 ceph-mon[106466]: failed to encode map e801 with expected crc 2024-03-29T05:07:28.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:27 smithi181 ceph-mon[106466]: osdmap e801: 8 total, 8 up, 8 in 2024-03-29T05:07:28.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:27 smithi181 ceph-mon[106466]: failed to encode map e801 with expected crc 2024-03-29T05:07:28.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:27 smithi181 ceph-mon[106466]: failed to encode map e801 with expected crc 2024-03-29T05:07:28.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:27 smithi181 ceph-mon[106466]: osdmap e802: 8 total, 8 up, 8 in 2024-03-29T05:07:29.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:28 smithi187 ceph-mon[93536]: failed to encode map e802 with expected crc 2024-03-29T05:07:29.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:28 smithi187 ceph-mon[93536]: failed to encode map e802 with expected crc 2024-03-29T05:07:29.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:28 smithi187 ceph-mon[93536]: failed to encode map e802 with expected crc 2024-03-29T05:07:29.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:28 smithi187 ceph-mon[93536]: pgmap v929: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 5.4 KiB/s wr, 124 op/s 2024-03-29T05:07:29.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:28 smithi187 ceph-mon[93536]: osdmap e803: 8 total, 8 up, 8 in 2024-03-29T05:07:29.208 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:07:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:07:28] "GET /metrics HTTP/1.1" 200 82156 "" "Prometheus/2.43.0" 2024-03-29T05:07:29.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:28 smithi181 ceph-mon[106466]: failed to encode map e802 with expected crc 2024-03-29T05:07:29.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:28 smithi181 ceph-mon[106466]: failed to encode map e802 with expected crc 2024-03-29T05:07:29.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:28 smithi181 ceph-mon[106466]: failed to encode map e802 with expected crc 2024-03-29T05:07:29.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:28 smithi181 ceph-mon[106466]: pgmap v929: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 5.4 KiB/s wr, 124 op/s 2024-03-29T05:07:29.208 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:28 smithi181 ceph-mon[106466]: osdmap e803: 8 total, 8 up, 8 in 2024-03-29T05:07:29.209 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:28 smithi181 ceph-mon[101970]: failed to encode map e802 with expected crc 2024-03-29T05:07:29.209 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:28 smithi181 ceph-mon[101970]: failed to encode map e802 with expected crc 2024-03-29T05:07:29.209 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:28 smithi181 ceph-mon[101970]: failed to encode map e802 with expected crc 2024-03-29T05:07:29.209 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:28 smithi181 ceph-mon[101970]: pgmap v929: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 5.4 KiB/s wr, 124 op/s 2024-03-29T05:07:29.209 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:28 smithi181 ceph-mon[101970]: osdmap e803: 8 total, 8 up, 8 in 2024-03-29T05:07:30.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:29 smithi187 ceph-mon[93536]: failed to encode map e803 with expected crc 2024-03-29T05:07:30.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:29 smithi187 ceph-mon[93536]: failed to encode map e803 with expected crc 2024-03-29T05:07:30.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:29 smithi187 ceph-mon[93536]: failed to encode map e803 with expected crc 2024-03-29T05:07:30.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:29 smithi187 ceph-mon[93536]: osdmap e804: 8 total, 8 up, 8 in 2024-03-29T05:07:30.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:29 smithi181 ceph-mon[101970]: failed to encode map e803 with expected crc 2024-03-29T05:07:30.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:29 smithi181 ceph-mon[101970]: failed to encode map e803 with expected crc 2024-03-29T05:07:30.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:29 smithi181 ceph-mon[101970]: failed to encode map e803 with expected crc 2024-03-29T05:07:30.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:29 smithi181 ceph-mon[101970]: osdmap e804: 8 total, 8 up, 8 in 2024-03-29T05:07:30.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:29 smithi181 ceph-mon[106466]: failed to encode map e803 with expected crc 2024-03-29T05:07:30.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:29 smithi181 ceph-mon[106466]: failed to encode map e803 with expected crc 2024-03-29T05:07:30.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:29 smithi181 ceph-mon[106466]: failed to encode map e803 with expected crc 2024-03-29T05:07:30.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:29 smithi181 ceph-mon[106466]: osdmap e804: 8 total, 8 up, 8 in 2024-03-29T05:07:31.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:30 smithi187 ceph-mon[93536]: pgmap v932: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 5.4 KiB/s wr, 125 op/s 2024-03-29T05:07:31.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:30 smithi187 ceph-mon[93536]: failed to encode map e804 with expected crc 2024-03-29T05:07:31.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:30 smithi187 ceph-mon[93536]: failed to encode map e804 with expected crc 2024-03-29T05:07:31.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:30 smithi187 ceph-mon[93536]: failed to encode map e804 with expected crc 2024-03-29T05:07:31.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:30 smithi187 ceph-mon[93536]: osdmap e805: 8 total, 8 up, 8 in 2024-03-29T05:07:31.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:30 smithi181 ceph-mon[106466]: pgmap v932: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 5.4 KiB/s wr, 125 op/s 2024-03-29T05:07:31.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:30 smithi181 ceph-mon[106466]: failed to encode map e804 with expected crc 2024-03-29T05:07:31.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:30 smithi181 ceph-mon[106466]: failed to encode map e804 with expected crc 2024-03-29T05:07:31.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:30 smithi181 ceph-mon[106466]: failed to encode map e804 with expected crc 2024-03-29T05:07:31.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:30 smithi181 ceph-mon[106466]: osdmap e805: 8 total, 8 up, 8 in 2024-03-29T05:07:31.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:30 smithi181 ceph-mon[101970]: pgmap v932: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 5.4 KiB/s wr, 125 op/s 2024-03-29T05:07:31.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:30 smithi181 ceph-mon[101970]: failed to encode map e804 with expected crc 2024-03-29T05:07:31.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:30 smithi181 ceph-mon[101970]: failed to encode map e804 with expected crc 2024-03-29T05:07:31.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:30 smithi181 ceph-mon[101970]: failed to encode map e804 with expected crc 2024-03-29T05:07:31.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:30 smithi181 ceph-mon[101970]: osdmap e805: 8 total, 8 up, 8 in 2024-03-29T05:07:32.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:31 smithi187 ceph-mon[93536]: failed to encode map e805 with expected crc 2024-03-29T05:07:32.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:31 smithi187 ceph-mon[93536]: failed to encode map e805 with expected crc 2024-03-29T05:07:32.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:31 smithi187 ceph-mon[93536]: failed to encode map e805 with expected crc 2024-03-29T05:07:32.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:31 smithi187 ceph-mon[93536]: osdmap e806: 8 total, 8 up, 8 in 2024-03-29T05:07:32.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:31 smithi181 ceph-mon[101970]: failed to encode map e805 with expected crc 2024-03-29T05:07:32.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:31 smithi181 ceph-mon[101970]: failed to encode map e805 with expected crc 2024-03-29T05:07:32.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:31 smithi181 ceph-mon[101970]: failed to encode map e805 with expected crc 2024-03-29T05:07:32.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:31 smithi181 ceph-mon[101970]: osdmap e806: 8 total, 8 up, 8 in 2024-03-29T05:07:32.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:31 smithi181 ceph-mon[106466]: failed to encode map e805 with expected crc 2024-03-29T05:07:32.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:31 smithi181 ceph-mon[106466]: failed to encode map e805 with expected crc 2024-03-29T05:07:32.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:31 smithi181 ceph-mon[106466]: failed to encode map e805 with expected crc 2024-03-29T05:07:32.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:31 smithi181 ceph-mon[106466]: osdmap e806: 8 total, 8 up, 8 in 2024-03-29T05:07:33.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:32 smithi187 ceph-mon[93536]: failed to encode map e806 with expected crc 2024-03-29T05:07:33.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:32 smithi187 ceph-mon[93536]: pgmap v935: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 3.2 KiB/s wr, 15 op/s 2024-03-29T05:07:33.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:32 smithi187 ceph-mon[93536]: failed to encode map e806 with expected crc 2024-03-29T05:07:33.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:32 smithi187 ceph-mon[93536]: failed to encode map e806 with expected crc 2024-03-29T05:07:33.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:32 smithi187 ceph-mon[93536]: osdmap e807: 8 total, 8 up, 8 in 2024-03-29T05:07:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:32 smithi181 ceph-mon[106466]: failed to encode map e806 with expected crc 2024-03-29T05:07:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:32 smithi181 ceph-mon[106466]: pgmap v935: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 3.2 KiB/s wr, 15 op/s 2024-03-29T05:07:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:32 smithi181 ceph-mon[106466]: failed to encode map e806 with expected crc 2024-03-29T05:07:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:32 smithi181 ceph-mon[106466]: failed to encode map e806 with expected crc 2024-03-29T05:07:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:32 smithi181 ceph-mon[106466]: osdmap e807: 8 total, 8 up, 8 in 2024-03-29T05:07:33.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:32 smithi181 ceph-mon[101970]: failed to encode map e806 with expected crc 2024-03-29T05:07:33.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:32 smithi181 ceph-mon[101970]: pgmap v935: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 3.2 KiB/s wr, 15 op/s 2024-03-29T05:07:33.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:32 smithi181 ceph-mon[101970]: failed to encode map e806 with expected crc 2024-03-29T05:07:33.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:32 smithi181 ceph-mon[101970]: failed to encode map e806 with expected crc 2024-03-29T05:07:33.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:32 smithi181 ceph-mon[101970]: osdmap e807: 8 total, 8 up, 8 in 2024-03-29T05:07:34.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:33 smithi187 ceph-mon[93536]: failed to encode map e807 with expected crc 2024-03-29T05:07:34.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:33 smithi187 ceph-mon[93536]: failed to encode map e807 with expected crc 2024-03-29T05:07:34.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:33 smithi187 ceph-mon[93536]: failed to encode map e807 with expected crc 2024-03-29T05:07:34.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:33 smithi187 ceph-mon[93536]: osdmap e808: 8 total, 8 up, 8 in 2024-03-29T05:07:34.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:33 smithi181 ceph-mon[106466]: failed to encode map e807 with expected crc 2024-03-29T05:07:34.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:33 smithi181 ceph-mon[106466]: failed to encode map e807 with expected crc 2024-03-29T05:07:34.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:33 smithi181 ceph-mon[106466]: failed to encode map e807 with expected crc 2024-03-29T05:07:34.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:33 smithi181 ceph-mon[106466]: osdmap e808: 8 total, 8 up, 8 in 2024-03-29T05:07:34.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:33 smithi181 ceph-mon[101970]: failed to encode map e807 with expected crc 2024-03-29T05:07:34.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:33 smithi181 ceph-mon[101970]: failed to encode map e807 with expected crc 2024-03-29T05:07:34.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:33 smithi181 ceph-mon[101970]: failed to encode map e807 with expected crc 2024-03-29T05:07:34.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:33 smithi181 ceph-mon[101970]: osdmap e808: 8 total, 8 up, 8 in 2024-03-29T05:07:35.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:34 smithi187 ceph-mon[93536]: failed to encode map e808 with expected crc 2024-03-29T05:07:35.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:34 smithi187 ceph-mon[93536]: pgmap v938: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 3.5 KiB/s wr, 19 op/s 2024-03-29T05:07:35.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:34 smithi187 ceph-mon[93536]: failed to encode map e808 with expected crc 2024-03-29T05:07:35.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:34 smithi187 ceph-mon[93536]: osdmap e809: 8 total, 8 up, 8 in 2024-03-29T05:07:35.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:34 smithi181 ceph-mon[106466]: failed to encode map e808 with expected crc 2024-03-29T05:07:35.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:34 smithi181 ceph-mon[106466]: pgmap v938: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 3.5 KiB/s wr, 19 op/s 2024-03-29T05:07:35.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:34 smithi181 ceph-mon[106466]: failed to encode map e808 with expected crc 2024-03-29T05:07:35.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:34 smithi181 ceph-mon[106466]: osdmap e809: 8 total, 8 up, 8 in 2024-03-29T05:07:35.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:34 smithi181 ceph-mon[101970]: failed to encode map e808 with expected crc 2024-03-29T05:07:35.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:34 smithi181 ceph-mon[101970]: pgmap v938: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 3.5 KiB/s wr, 19 op/s 2024-03-29T05:07:35.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:34 smithi181 ceph-mon[101970]: failed to encode map e808 with expected crc 2024-03-29T05:07:35.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:34 smithi181 ceph-mon[101970]: osdmap e809: 8 total, 8 up, 8 in 2024-03-29T05:07:36.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:35 smithi187 ceph-mon[93536]: failed to encode map e809 with expected crc 2024-03-29T05:07:36.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:35 smithi187 ceph-mon[93536]: failed to encode map e809 with expected crc 2024-03-29T05:07:36.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:35 smithi187 ceph-mon[93536]: osdmap e810: 8 total, 8 up, 8 in 2024-03-29T05:07:36.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:35 smithi181 ceph-mon[106466]: failed to encode map e809 with expected crc 2024-03-29T05:07:36.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:35 smithi181 ceph-mon[106466]: failed to encode map e809 with expected crc 2024-03-29T05:07:36.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:35 smithi181 ceph-mon[106466]: osdmap e810: 8 total, 8 up, 8 in 2024-03-29T05:07:36.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:35 smithi181 ceph-mon[101970]: failed to encode map e809 with expected crc 2024-03-29T05:07:36.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:35 smithi181 ceph-mon[101970]: failed to encode map e809 with expected crc 2024-03-29T05:07:36.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:35 smithi181 ceph-mon[101970]: osdmap e810: 8 total, 8 up, 8 in 2024-03-29T05:07:37.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:36 smithi187 ceph-mon[93536]: pgmap v940: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.7 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-29T05:07:37.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:36 smithi187 ceph-mon[93536]: failed to encode map e810 with expected crc 2024-03-29T05:07:37.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:36 smithi187 ceph-mon[93536]: failed to encode map e810 with expected crc 2024-03-29T05:07:37.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:36 smithi187 ceph-mon[93536]: failed to encode map e810 with expected crc 2024-03-29T05:07:37.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:36 smithi187 ceph-mon[93536]: osdmap e811: 8 total, 8 up, 8 in 2024-03-29T05:07:37.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:36 smithi181 ceph-mon[101970]: pgmap v940: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.7 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-29T05:07:37.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:36 smithi181 ceph-mon[101970]: failed to encode map e810 with expected crc 2024-03-29T05:07:37.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:36 smithi181 ceph-mon[101970]: failed to encode map e810 with expected crc 2024-03-29T05:07:37.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:36 smithi181 ceph-mon[101970]: failed to encode map e810 with expected crc 2024-03-29T05:07:37.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:36 smithi181 ceph-mon[101970]: osdmap e811: 8 total, 8 up, 8 in 2024-03-29T05:07:37.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:36 smithi181 ceph-mon[106466]: pgmap v940: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.7 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-29T05:07:37.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:36 smithi181 ceph-mon[106466]: failed to encode map e810 with expected crc 2024-03-29T05:07:37.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:36 smithi181 ceph-mon[106466]: failed to encode map e810 with expected crc 2024-03-29T05:07:37.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:36 smithi181 ceph-mon[106466]: failed to encode map e810 with expected crc 2024-03-29T05:07:37.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:36 smithi181 ceph-mon[106466]: osdmap e811: 8 total, 8 up, 8 in 2024-03-29T05:07:38.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:37 smithi187 ceph-mon[93536]: failed to encode map e811 with expected crc 2024-03-29T05:07:38.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:37 smithi187 ceph-mon[93536]: failed to encode map e811 with expected crc 2024-03-29T05:07:38.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:37 smithi187 ceph-mon[93536]: failed to encode map e811 with expected crc 2024-03-29T05:07:38.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:37 smithi187 ceph-mon[93536]: osdmap e812: 8 total, 8 up, 8 in 2024-03-29T05:07:38.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:37 smithi181 ceph-mon[101970]: failed to encode map e811 with expected crc 2024-03-29T05:07:38.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:37 smithi181 ceph-mon[101970]: failed to encode map e811 with expected crc 2024-03-29T05:07:38.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:37 smithi181 ceph-mon[101970]: failed to encode map e811 with expected crc 2024-03-29T05:07:38.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:37 smithi181 ceph-mon[101970]: osdmap e812: 8 total, 8 up, 8 in 2024-03-29T05:07:38.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:37 smithi181 ceph-mon[106466]: failed to encode map e811 with expected crc 2024-03-29T05:07:38.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:37 smithi181 ceph-mon[106466]: failed to encode map e811 with expected crc 2024-03-29T05:07:38.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:37 smithi181 ceph-mon[106466]: failed to encode map e811 with expected crc 2024-03-29T05:07:38.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:37 smithi181 ceph-mon[106466]: osdmap e812: 8 total, 8 up, 8 in 2024-03-29T05:07:39.106 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:07:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:07:38] "GET /metrics HTTP/1.1" 200 82156 "" "Prometheus/2.43.0" 2024-03-29T05:07:39.107 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:38 smithi181 ceph-mon[106466]: pgmap v943: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 5.5 KiB/s wr, 244 op/s 2024-03-29T05:07:39.107 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:38 smithi181 ceph-mon[106466]: failed to encode map e812 with expected crc 2024-03-29T05:07:39.107 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:38 smithi181 ceph-mon[106466]: failed to encode map e812 with expected crc 2024-03-29T05:07:39.107 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:38 smithi181 ceph-mon[106466]: failed to encode map e812 with expected crc 2024-03-29T05:07:39.107 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:38 smithi181 ceph-mon[106466]: osdmap e813: 8 total, 8 up, 8 in 2024-03-29T05:07:39.107 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:38 smithi181 ceph-mon[101970]: pgmap v943: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 5.5 KiB/s wr, 244 op/s 2024-03-29T05:07:39.107 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:38 smithi181 ceph-mon[101970]: failed to encode map e812 with expected crc 2024-03-29T05:07:39.107 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:38 smithi181 ceph-mon[101970]: failed to encode map e812 with expected crc 2024-03-29T05:07:39.107 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:38 smithi181 ceph-mon[101970]: failed to encode map e812 with expected crc 2024-03-29T05:07:39.107 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:38 smithi181 ceph-mon[101970]: osdmap e813: 8 total, 8 up, 8 in 2024-03-29T05:07:39.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:38 smithi187 ceph-mon[93536]: pgmap v943: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 5.5 KiB/s wr, 244 op/s 2024-03-29T05:07:39.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:38 smithi187 ceph-mon[93536]: failed to encode map e812 with expected crc 2024-03-29T05:07:39.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:38 smithi187 ceph-mon[93536]: failed to encode map e812 with expected crc 2024-03-29T05:07:39.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:38 smithi187 ceph-mon[93536]: failed to encode map e812 with expected crc 2024-03-29T05:07:39.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:38 smithi187 ceph-mon[93536]: osdmap e813: 8 total, 8 up, 8 in 2024-03-29T05:07:40.133 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:39 smithi181 ceph-mon[101970]: failed to encode map e813 with expected crc 2024-03-29T05:07:40.133 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:39 smithi181 ceph-mon[101970]: failed to encode map e813 with expected crc 2024-03-29T05:07:40.133 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:39 smithi181 ceph-mon[101970]: failed to encode map e813 with expected crc 2024-03-29T05:07:40.133 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:39 smithi181 ceph-mon[101970]: osdmap e814: 8 total, 8 up, 8 in 2024-03-29T05:07:40.133 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:39 smithi181 ceph-mon[106466]: failed to encode map e813 with expected crc 2024-03-29T05:07:40.133 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:39 smithi181 ceph-mon[106466]: failed to encode map e813 with expected crc 2024-03-29T05:07:40.133 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:39 smithi181 ceph-mon[106466]: failed to encode map e813 with expected crc 2024-03-29T05:07:40.134 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:39 smithi181 ceph-mon[106466]: osdmap e814: 8 total, 8 up, 8 in 2024-03-29T05:07:40.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:39 smithi187 ceph-mon[93536]: failed to encode map e813 with expected crc 2024-03-29T05:07:40.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:39 smithi187 ceph-mon[93536]: failed to encode map e813 with expected crc 2024-03-29T05:07:40.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:39 smithi187 ceph-mon[93536]: failed to encode map e813 with expected crc 2024-03-29T05:07:40.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:39 smithi187 ceph-mon[93536]: osdmap e814: 8 total, 8 up, 8 in 2024-03-29T05:07:41.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:40 smithi187 ceph-mon[93536]: pgmap v946: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 5.5 KiB/s wr, 244 op/s 2024-03-29T05:07:41.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:40 smithi187 ceph-mon[93536]: failed to encode map e814 with expected crc 2024-03-29T05:07:41.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:40 smithi187 ceph-mon[93536]: failed to encode map e814 with expected crc 2024-03-29T05:07:41.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:40 smithi187 ceph-mon[93536]: failed to encode map e814 with expected crc 2024-03-29T05:07:41.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:40 smithi187 ceph-mon[93536]: osdmap e815: 8 total, 8 up, 8 in 2024-03-29T05:07:41.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:40 smithi181 ceph-mon[106466]: pgmap v946: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 5.5 KiB/s wr, 244 op/s 2024-03-29T05:07:41.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:40 smithi181 ceph-mon[106466]: failed to encode map e814 with expected crc 2024-03-29T05:07:41.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:40 smithi181 ceph-mon[106466]: failed to encode map e814 with expected crc 2024-03-29T05:07:41.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:40 smithi181 ceph-mon[106466]: failed to encode map e814 with expected crc 2024-03-29T05:07:41.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:40 smithi181 ceph-mon[106466]: osdmap e815: 8 total, 8 up, 8 in 2024-03-29T05:07:41.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:40 smithi181 ceph-mon[101970]: pgmap v946: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 5.5 KiB/s wr, 244 op/s 2024-03-29T05:07:41.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:40 smithi181 ceph-mon[101970]: failed to encode map e814 with expected crc 2024-03-29T05:07:41.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:40 smithi181 ceph-mon[101970]: failed to encode map e814 with expected crc 2024-03-29T05:07:41.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:40 smithi181 ceph-mon[101970]: failed to encode map e814 with expected crc 2024-03-29T05:07:41.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:40 smithi181 ceph-mon[101970]: osdmap e815: 8 total, 8 up, 8 in 2024-03-29T05:07:42.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:41 smithi187 ceph-mon[93536]: failed to encode map e815 with expected crc 2024-03-29T05:07:42.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:41 smithi187 ceph-mon[93536]: failed to encode map e815 with expected crc 2024-03-29T05:07:42.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:41 smithi187 ceph-mon[93536]: failed to encode map e815 with expected crc 2024-03-29T05:07:42.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:07:42.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:41 smithi187 ceph-mon[93536]: osdmap e816: 8 total, 8 up, 8 in 2024-03-29T05:07:42.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:41 smithi181 ceph-mon[106466]: failed to encode map e815 with expected crc 2024-03-29T05:07:42.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:41 smithi181 ceph-mon[106466]: failed to encode map e815 with expected crc 2024-03-29T05:07:42.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:41 smithi181 ceph-mon[106466]: failed to encode map e815 with expected crc 2024-03-29T05:07:42.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:07:42.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:41 smithi181 ceph-mon[106466]: osdmap e816: 8 total, 8 up, 8 in 2024-03-29T05:07:42.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:41 smithi181 ceph-mon[101970]: failed to encode map e815 with expected crc 2024-03-29T05:07:42.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:41 smithi181 ceph-mon[101970]: failed to encode map e815 with expected crc 2024-03-29T05:07:42.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:41 smithi181 ceph-mon[101970]: failed to encode map e815 with expected crc 2024-03-29T05:07:42.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:07:42.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:41 smithi181 ceph-mon[101970]: osdmap e816: 8 total, 8 up, 8 in 2024-03-29T05:07:43.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:42 smithi187 ceph-mon[93536]: pgmap v949: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 25 op/s 2024-03-29T05:07:43.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:42 smithi187 ceph-mon[93536]: failed to encode map e816 with expected crc 2024-03-29T05:07:43.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:42 smithi187 ceph-mon[93536]: failed to encode map e816 with expected crc 2024-03-29T05:07:43.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:42 smithi187 ceph-mon[93536]: failed to encode map e816 with expected crc 2024-03-29T05:07:43.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:42 smithi187 ceph-mon[93536]: osdmap e817: 8 total, 8 up, 8 in 2024-03-29T05:07:43.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:42 smithi181 ceph-mon[106466]: pgmap v949: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 25 op/s 2024-03-29T05:07:43.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:42 smithi181 ceph-mon[106466]: failed to encode map e816 with expected crc 2024-03-29T05:07:43.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:42 smithi181 ceph-mon[106466]: failed to encode map e816 with expected crc 2024-03-29T05:07:43.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:42 smithi181 ceph-mon[106466]: failed to encode map e816 with expected crc 2024-03-29T05:07:43.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:42 smithi181 ceph-mon[106466]: osdmap e817: 8 total, 8 up, 8 in 2024-03-29T05:07:43.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:42 smithi181 ceph-mon[101970]: pgmap v949: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 25 op/s 2024-03-29T05:07:43.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:42 smithi181 ceph-mon[101970]: failed to encode map e816 with expected crc 2024-03-29T05:07:43.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:42 smithi181 ceph-mon[101970]: failed to encode map e816 with expected crc 2024-03-29T05:07:43.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:42 smithi181 ceph-mon[101970]: failed to encode map e816 with expected crc 2024-03-29T05:07:43.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:42 smithi181 ceph-mon[101970]: osdmap e817: 8 total, 8 up, 8 in 2024-03-29T05:07:44.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:43 smithi187 ceph-mon[93536]: failed to encode map e817 with expected crc 2024-03-29T05:07:44.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:43 smithi187 ceph-mon[93536]: failed to encode map e817 with expected crc 2024-03-29T05:07:44.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:43 smithi187 ceph-mon[93536]: failed to encode map e817 with expected crc 2024-03-29T05:07:44.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:43 smithi187 ceph-mon[93536]: failed to encode map e817 with expected crc 2024-03-29T05:07:44.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:43 smithi187 ceph-mon[93536]: osdmap e818: 8 total, 8 up, 8 in 2024-03-29T05:07:44.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:43 smithi181 ceph-mon[106466]: failed to encode map e817 with expected crc 2024-03-29T05:07:44.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:43 smithi181 ceph-mon[106466]: failed to encode map e817 with expected crc 2024-03-29T05:07:44.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:43 smithi181 ceph-mon[106466]: failed to encode map e817 with expected crc 2024-03-29T05:07:44.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:43 smithi181 ceph-mon[106466]: failed to encode map e817 with expected crc 2024-03-29T05:07:44.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:43 smithi181 ceph-mon[106466]: osdmap e818: 8 total, 8 up, 8 in 2024-03-29T05:07:44.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:43 smithi181 ceph-mon[101970]: failed to encode map e817 with expected crc 2024-03-29T05:07:44.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:43 smithi181 ceph-mon[101970]: failed to encode map e817 with expected crc 2024-03-29T05:07:44.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:43 smithi181 ceph-mon[101970]: failed to encode map e817 with expected crc 2024-03-29T05:07:44.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:43 smithi181 ceph-mon[101970]: failed to encode map e817 with expected crc 2024-03-29T05:07:44.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:43 smithi181 ceph-mon[101970]: osdmap e818: 8 total, 8 up, 8 in 2024-03-29T05:07:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:44 smithi187 ceph-mon[93536]: failed to encode map e818 with expected crc 2024-03-29T05:07:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:44 smithi187 ceph-mon[93536]: failed to encode map e818 with expected crc 2024-03-29T05:07:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:44 smithi187 ceph-mon[93536]: failed to encode map e818 with expected crc 2024-03-29T05:07:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:44 smithi187 ceph-mon[93536]: pgmap v953: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 5.0 KiB/s wr, 28 op/s 2024-03-29T05:07:45.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:44 smithi187 ceph-mon[93536]: osdmap e819: 8 total, 8 up, 8 in 2024-03-29T05:07:45.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:44 smithi181 ceph-mon[106466]: failed to encode map e818 with expected crc 2024-03-29T05:07:45.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:44 smithi181 ceph-mon[106466]: failed to encode map e818 with expected crc 2024-03-29T05:07:45.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:44 smithi181 ceph-mon[106466]: failed to encode map e818 with expected crc 2024-03-29T05:07:45.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:44 smithi181 ceph-mon[106466]: pgmap v953: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 5.0 KiB/s wr, 28 op/s 2024-03-29T05:07:45.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:44 smithi181 ceph-mon[106466]: osdmap e819: 8 total, 8 up, 8 in 2024-03-29T05:07:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:44 smithi181 ceph-mon[101970]: failed to encode map e818 with expected crc 2024-03-29T05:07:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:44 smithi181 ceph-mon[101970]: failed to encode map e818 with expected crc 2024-03-29T05:07:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:44 smithi181 ceph-mon[101970]: failed to encode map e818 with expected crc 2024-03-29T05:07:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:44 smithi181 ceph-mon[101970]: pgmap v953: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 5.0 KiB/s wr, 28 op/s 2024-03-29T05:07:45.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:44 smithi181 ceph-mon[101970]: osdmap e819: 8 total, 8 up, 8 in 2024-03-29T05:07:46.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:45 smithi187 ceph-mon[93536]: failed to encode map e819 with expected crc 2024-03-29T05:07:46.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:45 smithi187 ceph-mon[93536]: failed to encode map e819 with expected crc 2024-03-29T05:07:46.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:45 smithi187 ceph-mon[93536]: failed to encode map e819 with expected crc 2024-03-29T05:07:46.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:45 smithi187 ceph-mon[93536]: osdmap e820: 8 total, 8 up, 8 in 2024-03-29T05:07:46.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:45 smithi181 ceph-mon[101970]: failed to encode map e819 with expected crc 2024-03-29T05:07:46.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:45 smithi181 ceph-mon[101970]: failed to encode map e819 with expected crc 2024-03-29T05:07:46.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:45 smithi181 ceph-mon[101970]: failed to encode map e819 with expected crc 2024-03-29T05:07:46.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:45 smithi181 ceph-mon[101970]: osdmap e820: 8 total, 8 up, 8 in 2024-03-29T05:07:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:45 smithi181 ceph-mon[106466]: failed to encode map e819 with expected crc 2024-03-29T05:07:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:45 smithi181 ceph-mon[106466]: failed to encode map e819 with expected crc 2024-03-29T05:07:46.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:45 smithi181 ceph-mon[106466]: failed to encode map e819 with expected crc 2024-03-29T05:07:46.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:45 smithi181 ceph-mon[106466]: osdmap e820: 8 total, 8 up, 8 in 2024-03-29T05:07:47.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:46 smithi187 ceph-mon[93536]: failed to encode map e820 with expected crc 2024-03-29T05:07:47.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:46 smithi187 ceph-mon[93536]: pgmap v956: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 3 op/s 2024-03-29T05:07:47.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:46 smithi187 ceph-mon[93536]: failed to encode map e820 with expected crc 2024-03-29T05:07:47.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:46 smithi187 ceph-mon[93536]: failed to encode map e820 with expected crc 2024-03-29T05:07:47.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:46 smithi187 ceph-mon[93536]: osdmap e821: 8 total, 8 up, 8 in 2024-03-29T05:07:47.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:46 smithi181 ceph-mon[101970]: failed to encode map e820 with expected crc 2024-03-29T05:07:47.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:46 smithi181 ceph-mon[101970]: pgmap v956: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 3 op/s 2024-03-29T05:07:47.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:46 smithi181 ceph-mon[101970]: failed to encode map e820 with expected crc 2024-03-29T05:07:47.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:46 smithi181 ceph-mon[101970]: failed to encode map e820 with expected crc 2024-03-29T05:07:47.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:46 smithi181 ceph-mon[101970]: osdmap e821: 8 total, 8 up, 8 in 2024-03-29T05:07:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:46 smithi181 ceph-mon[106466]: failed to encode map e820 with expected crc 2024-03-29T05:07:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:46 smithi181 ceph-mon[106466]: pgmap v956: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 3 op/s 2024-03-29T05:07:47.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:46 smithi181 ceph-mon[106466]: failed to encode map e820 with expected crc 2024-03-29T05:07:47.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:46 smithi181 ceph-mon[106466]: failed to encode map e820 with expected crc 2024-03-29T05:07:47.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:46 smithi181 ceph-mon[106466]: osdmap e821: 8 total, 8 up, 8 in 2024-03-29T05:07:47.797 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] TestLibRBD.ConcurentOperations (25800 ms) 2024-03-29T05:07:48.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:47 smithi187 ceph-mon[93536]: failed to encode map e821 with expected crc 2024-03-29T05:07:48.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:47 smithi187 ceph-mon[93536]: failed to encode map e821 with expected crc 2024-03-29T05:07:48.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:47 smithi187 ceph-mon[93536]: failed to encode map e821 with expected crc 2024-03-29T05:07:48.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:47 smithi187 ceph-mon[93536]: osdmap e822: 8 total, 8 up, 8 in 2024-03-29T05:07:48.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:47 smithi181 ceph-mon[101970]: failed to encode map e821 with expected crc 2024-03-29T05:07:48.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:47 smithi181 ceph-mon[101970]: failed to encode map e821 with expected crc 2024-03-29T05:07:48.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:47 smithi181 ceph-mon[101970]: failed to encode map e821 with expected crc 2024-03-29T05:07:48.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:47 smithi181 ceph-mon[101970]: osdmap e822: 8 total, 8 up, 8 in 2024-03-29T05:07:48.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:47 smithi181 ceph-mon[106466]: failed to encode map e821 with expected crc 2024-03-29T05:07:48.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:47 smithi181 ceph-mon[106466]: failed to encode map e821 with expected crc 2024-03-29T05:07:48.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:47 smithi181 ceph-mon[106466]: failed to encode map e821 with expected crc 2024-03-29T05:07:48.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:47 smithi181 ceph-mon[106466]: osdmap e822: 8 total, 8 up, 8 in 2024-03-29T05:07:49.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:48 smithi187 ceph-mon[93536]: failed to encode map e822 with expected crc 2024-03-29T05:07:49.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:48 smithi187 ceph-mon[93536]: failed to encode map e822 with expected crc 2024-03-29T05:07:49.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:48 smithi187 ceph-mon[93536]: failed to encode map e822 with expected crc 2024-03-29T05:07:49.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:48 smithi187 ceph-mon[93536]: pgmap v959: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 4.2 KiB/s wr, 87 op/s 2024-03-29T05:07:49.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:48 smithi187 ceph-mon[93536]: osdmap e823: 8 total, 8 up, 8 in 2024-03-29T05:07:49.205 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:07:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:07:48] "GET /metrics HTTP/1.1" 200 82151 "" "Prometheus/2.43.0" 2024-03-29T05:07:49.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:48 smithi181 ceph-mon[101970]: failed to encode map e822 with expected crc 2024-03-29T05:07:49.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:48 smithi181 ceph-mon[101970]: failed to encode map e822 with expected crc 2024-03-29T05:07:49.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:48 smithi181 ceph-mon[101970]: failed to encode map e822 with expected crc 2024-03-29T05:07:49.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:48 smithi181 ceph-mon[101970]: pgmap v959: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 4.2 KiB/s wr, 87 op/s 2024-03-29T05:07:49.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:48 smithi181 ceph-mon[101970]: osdmap e823: 8 total, 8 up, 8 in 2024-03-29T05:07:49.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:48 smithi181 ceph-mon[106466]: failed to encode map e822 with expected crc 2024-03-29T05:07:49.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:48 smithi181 ceph-mon[106466]: failed to encode map e822 with expected crc 2024-03-29T05:07:49.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:48 smithi181 ceph-mon[106466]: failed to encode map e822 with expected crc 2024-03-29T05:07:49.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:48 smithi181 ceph-mon[106466]: pgmap v959: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 4.2 KiB/s wr, 87 op/s 2024-03-29T05:07:49.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:48 smithi181 ceph-mon[106466]: osdmap e823: 8 total, 8 up, 8 in 2024-03-29T05:07:50.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:49 smithi187 ceph-mon[93536]: failed to encode map e823 with expected crc 2024-03-29T05:07:50.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:49 smithi187 ceph-mon[93536]: failed to encode map e823 with expected crc 2024-03-29T05:07:50.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:49 smithi187 ceph-mon[93536]: failed to encode map e823 with expected crc 2024-03-29T05:07:50.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:49 smithi187 ceph-mon[93536]: osdmap e824: 8 total, 8 up, 8 in 2024-03-29T05:07:50.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:49 smithi181 ceph-mon[101970]: failed to encode map e823 with expected crc 2024-03-29T05:07:50.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:49 smithi181 ceph-mon[101970]: failed to encode map e823 with expected crc 2024-03-29T05:07:50.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:49 smithi181 ceph-mon[101970]: failed to encode map e823 with expected crc 2024-03-29T05:07:50.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:49 smithi181 ceph-mon[101970]: osdmap e824: 8 total, 8 up, 8 in 2024-03-29T05:07:50.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:49 smithi181 ceph-mon[106466]: failed to encode map e823 with expected crc 2024-03-29T05:07:50.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:49 smithi181 ceph-mon[106466]: failed to encode map e823 with expected crc 2024-03-29T05:07:50.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:49 smithi181 ceph-mon[106466]: failed to encode map e823 with expected crc 2024-03-29T05:07:50.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:49 smithi181 ceph-mon[106466]: osdmap e824: 8 total, 8 up, 8 in 2024-03-29T05:07:51.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:50 smithi187 ceph-mon[93536]: failed to encode map e824 with expected crc 2024-03-29T05:07:51.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:50 smithi187 ceph-mon[93536]: pgmap v962: 649 pgs: 649 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 14 op/s 2024-03-29T05:07:51.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:50 smithi187 ceph-mon[93536]: failed to encode map e824 with expected crc 2024-03-29T05:07:51.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:50 smithi187 ceph-mon[93536]: osdmap e825: 8 total, 8 up, 8 in 2024-03-29T05:07:51.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:50 smithi181 ceph-mon[101970]: failed to encode map e824 with expected crc 2024-03-29T05:07:51.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:50 smithi181 ceph-mon[101970]: pgmap v962: 649 pgs: 649 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 14 op/s 2024-03-29T05:07:51.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:50 smithi181 ceph-mon[101970]: failed to encode map e824 with expected crc 2024-03-29T05:07:51.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:50 smithi181 ceph-mon[101970]: osdmap e825: 8 total, 8 up, 8 in 2024-03-29T05:07:51.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:50 smithi181 ceph-mon[106466]: failed to encode map e824 with expected crc 2024-03-29T05:07:51.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:50 smithi181 ceph-mon[106466]: pgmap v962: 649 pgs: 649 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 14 op/s 2024-03-29T05:07:51.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:50 smithi181 ceph-mon[106466]: failed to encode map e824 with expected crc 2024-03-29T05:07:51.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:50 smithi181 ceph-mon[106466]: osdmap e825: 8 total, 8 up, 8 in 2024-03-29T05:07:52.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:51 smithi187 ceph-mon[93536]: failed to encode map e824 with expected crc 2024-03-29T05:07:52.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:51 smithi187 ceph-mon[93536]: failed to encode map e825 with expected crc 2024-03-29T05:07:52.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:51 smithi187 ceph-mon[93536]: osdmap e826: 8 total, 8 up, 8 in 2024-03-29T05:07:52.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:51 smithi181 ceph-mon[106466]: failed to encode map e824 with expected crc 2024-03-29T05:07:52.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:51 smithi181 ceph-mon[106466]: failed to encode map e825 with expected crc 2024-03-29T05:07:52.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:51 smithi181 ceph-mon[106466]: osdmap e826: 8 total, 8 up, 8 in 2024-03-29T05:07:52.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:51 smithi181 ceph-mon[101970]: failed to encode map e824 with expected crc 2024-03-29T05:07:52.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:51 smithi181 ceph-mon[101970]: failed to encode map e825 with expected crc 2024-03-29T05:07:52.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:51 smithi181 ceph-mon[101970]: osdmap e826: 8 total, 8 up, 8 in 2024-03-29T05:07:53.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:52 smithi187 ceph-mon[93536]: failed to encode map e826 with expected crc 2024-03-29T05:07:53.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:52 smithi187 ceph-mon[93536]: pgmap v965: 585 pgs: 585 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:53.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:52 smithi187 ceph-mon[93536]: failed to encode map e826 with expected crc 2024-03-29T05:07:53.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:52 smithi187 ceph-mon[93536]: osdmap e827: 8 total, 8 up, 8 in 2024-03-29T05:07:53.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:52 smithi181 ceph-mon[106466]: failed to encode map e826 with expected crc 2024-03-29T05:07:53.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:52 smithi181 ceph-mon[106466]: pgmap v965: 585 pgs: 585 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:53.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:52 smithi181 ceph-mon[106466]: failed to encode map e826 with expected crc 2024-03-29T05:07:53.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:52 smithi181 ceph-mon[106466]: osdmap e827: 8 total, 8 up, 8 in 2024-03-29T05:07:53.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:52 smithi181 ceph-mon[101970]: failed to encode map e826 with expected crc 2024-03-29T05:07:53.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:52 smithi181 ceph-mon[101970]: pgmap v965: 585 pgs: 585 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:53.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:52 smithi181 ceph-mon[101970]: failed to encode map e826 with expected crc 2024-03-29T05:07:53.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:52 smithi181 ceph-mon[101970]: osdmap e827: 8 total, 8 up, 8 in 2024-03-29T05:07:54.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:53 smithi187 ceph-mon[93536]: failed to encode map e826 with expected crc 2024-03-29T05:07:54.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:53 smithi187 ceph-mon[93536]: failed to encode map e827 with expected crc 2024-03-29T05:07:54.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:53 smithi187 ceph-mon[93536]: failed to encode map e827 with expected crc 2024-03-29T05:07:54.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:53 smithi187 ceph-mon[93536]: failed to encode map e827 with expected crc 2024-03-29T05:07:54.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:53 smithi187 ceph-mon[93536]: osdmap e828: 8 total, 8 up, 8 in 2024-03-29T05:07:54.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:53 smithi181 ceph-mon[106466]: failed to encode map e826 with expected crc 2024-03-29T05:07:54.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:53 smithi181 ceph-mon[106466]: failed to encode map e827 with expected crc 2024-03-29T05:07:54.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:53 smithi181 ceph-mon[106466]: failed to encode map e827 with expected crc 2024-03-29T05:07:54.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:53 smithi181 ceph-mon[106466]: failed to encode map e827 with expected crc 2024-03-29T05:07:54.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:53 smithi181 ceph-mon[106466]: osdmap e828: 8 total, 8 up, 8 in 2024-03-29T05:07:54.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:53 smithi181 ceph-mon[101970]: failed to encode map e826 with expected crc 2024-03-29T05:07:54.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:53 smithi181 ceph-mon[101970]: failed to encode map e827 with expected crc 2024-03-29T05:07:54.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:53 smithi181 ceph-mon[101970]: failed to encode map e827 with expected crc 2024-03-29T05:07:54.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:53 smithi181 ceph-mon[101970]: failed to encode map e827 with expected crc 2024-03-29T05:07:54.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:53 smithi181 ceph-mon[101970]: osdmap e828: 8 total, 8 up, 8 in 2024-03-29T05:07:55.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:54 smithi187 ceph-mon[93536]: failed to encode map e827 with expected crc 2024-03-29T05:07:55.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:54 smithi187 ceph-mon[93536]: failed to encode map e828 with expected crc 2024-03-29T05:07:55.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:54 smithi187 ceph-mon[93536]: pgmap v968: 521 pgs: 521 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:55.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:54 smithi187 ceph-mon[93536]: failed to encode map e828 with expected crc 2024-03-29T05:07:55.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:54 smithi187 ceph-mon[93536]: failed to encode map e828 with expected crc 2024-03-29T05:07:55.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:54 smithi187 ceph-mon[93536]: osdmap e829: 8 total, 8 up, 8 in 2024-03-29T05:07:55.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:54 smithi181 ceph-mon[101970]: failed to encode map e827 with expected crc 2024-03-29T05:07:55.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:54 smithi181 ceph-mon[101970]: failed to encode map e828 with expected crc 2024-03-29T05:07:55.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:54 smithi181 ceph-mon[101970]: pgmap v968: 521 pgs: 521 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:55.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:54 smithi181 ceph-mon[101970]: failed to encode map e828 with expected crc 2024-03-29T05:07:55.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:54 smithi181 ceph-mon[101970]: failed to encode map e828 with expected crc 2024-03-29T05:07:55.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:54 smithi181 ceph-mon[101970]: osdmap e829: 8 total, 8 up, 8 in 2024-03-29T05:07:55.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:54 smithi181 ceph-mon[106466]: failed to encode map e827 with expected crc 2024-03-29T05:07:55.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:54 smithi181 ceph-mon[106466]: failed to encode map e828 with expected crc 2024-03-29T05:07:55.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:54 smithi181 ceph-mon[106466]: pgmap v968: 521 pgs: 521 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:55.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:54 smithi181 ceph-mon[106466]: failed to encode map e828 with expected crc 2024-03-29T05:07:55.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:54 smithi181 ceph-mon[106466]: failed to encode map e828 with expected crc 2024-03-29T05:07:55.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:54 smithi181 ceph-mon[106466]: osdmap e829: 8 total, 8 up, 8 in 2024-03-29T05:07:56.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:56 smithi181 ceph-mon[101970]: failed to encode map e829 with expected crc 2024-03-29T05:07:56.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:56 smithi181 ceph-mon[101970]: osdmap e830: 8 total, 8 up, 8 in 2024-03-29T05:07:56.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:56 smithi181 ceph-mon[101970]: pgmap v971: 457 pgs: 457 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:56.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:56 smithi181 ceph-mon[101970]: failed to encode map e829 with expected crc 2024-03-29T05:07:56.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:56 smithi181 ceph-mon[101970]: failed to encode map e829 with expected crc 2024-03-29T05:07:56.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:07:56.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:56 smithi181 ceph-mon[106466]: failed to encode map e829 with expected crc 2024-03-29T05:07:56.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:56 smithi181 ceph-mon[106466]: osdmap e830: 8 total, 8 up, 8 in 2024-03-29T05:07:56.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:56 smithi181 ceph-mon[106466]: pgmap v971: 457 pgs: 457 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:56.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:56 smithi181 ceph-mon[106466]: failed to encode map e829 with expected crc 2024-03-29T05:07:56.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:56 smithi181 ceph-mon[106466]: failed to encode map e829 with expected crc 2024-03-29T05:07:56.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:07:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:56 smithi187 ceph-mon[93536]: failed to encode map e829 with expected crc 2024-03-29T05:07:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:56 smithi187 ceph-mon[93536]: osdmap e830: 8 total, 8 up, 8 in 2024-03-29T05:07:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:56 smithi187 ceph-mon[93536]: pgmap v971: 457 pgs: 457 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:56 smithi187 ceph-mon[93536]: failed to encode map e829 with expected crc 2024-03-29T05:07:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:56 smithi187 ceph-mon[93536]: failed to encode map e829 with expected crc 2024-03-29T05:07:57.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:07:58.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:57 smithi187 ceph-mon[93536]: failed to encode map e831 with expected crc 2024-03-29T05:07:58.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:57 smithi187 ceph-mon[93536]: osdmap e831: 8 total, 8 up, 8 in 2024-03-29T05:07:58.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:57 smithi187 ceph-mon[93536]: failed to encode map e831 with expected crc 2024-03-29T05:07:58.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:57 smithi187 ceph-mon[93536]: osdmap e832: 8 total, 8 up, 8 in 2024-03-29T05:07:58.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:57 smithi181 ceph-mon[101970]: failed to encode map e831 with expected crc 2024-03-29T05:07:58.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:57 smithi181 ceph-mon[101970]: osdmap e831: 8 total, 8 up, 8 in 2024-03-29T05:07:58.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:57 smithi181 ceph-mon[101970]: failed to encode map e831 with expected crc 2024-03-29T05:07:58.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:57 smithi181 ceph-mon[101970]: osdmap e832: 8 total, 8 up, 8 in 2024-03-29T05:07:58.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:57 smithi181 ceph-mon[106466]: failed to encode map e831 with expected crc 2024-03-29T05:07:58.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:57 smithi181 ceph-mon[106466]: osdmap e831: 8 total, 8 up, 8 in 2024-03-29T05:07:58.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:57 smithi181 ceph-mon[106466]: failed to encode map e831 with expected crc 2024-03-29T05:07:58.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:57 smithi181 ceph-mon[106466]: osdmap e832: 8 total, 8 up, 8 in 2024-03-29T05:07:59.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:58 smithi187 ceph-mon[93536]: failed to encode map e831 with expected crc 2024-03-29T05:07:59.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:58 smithi187 ceph-mon[93536]: pgmap v974: 393 pgs: 393 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:59.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:58 smithi187 ceph-mon[93536]: failed to encode map e832 with expected crc 2024-03-29T05:07:59.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:58 smithi187 ceph-mon[93536]: failed to encode map e832 with expected crc 2024-03-29T05:07:59.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:58 smithi187 ceph-mon[93536]: osdmap e833: 8 total, 8 up, 8 in 2024-03-29T05:07:59.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:07:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:07:58] "GET /metrics HTTP/1.1" 200 62906 "" "Prometheus/2.43.0" 2024-03-29T05:07:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:58 smithi181 ceph-mon[106466]: failed to encode map e831 with expected crc 2024-03-29T05:07:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:58 smithi181 ceph-mon[106466]: pgmap v974: 393 pgs: 393 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:58 smithi181 ceph-mon[106466]: failed to encode map e832 with expected crc 2024-03-29T05:07:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:58 smithi181 ceph-mon[106466]: failed to encode map e832 with expected crc 2024-03-29T05:07:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:58 smithi181 ceph-mon[106466]: osdmap e833: 8 total, 8 up, 8 in 2024-03-29T05:07:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:58 smithi181 ceph-mon[101970]: failed to encode map e831 with expected crc 2024-03-29T05:07:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:58 smithi181 ceph-mon[101970]: pgmap v974: 393 pgs: 393 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:07:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:58 smithi181 ceph-mon[101970]: failed to encode map e832 with expected crc 2024-03-29T05:07:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:58 smithi181 ceph-mon[101970]: failed to encode map e832 with expected crc 2024-03-29T05:07:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:58 smithi181 ceph-mon[101970]: osdmap e833: 8 total, 8 up, 8 in 2024-03-29T05:08:00.036 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:59 smithi187 ceph-mon[93536]: failed to encode map e832 with expected crc 2024-03-29T05:08:00.036 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:59 smithi187 ceph-mon[93536]: failed to encode map e833 with expected crc 2024-03-29T05:08:00.036 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:59 smithi187 ceph-mon[93536]: failed to encode map e833 with expected crc 2024-03-29T05:08:00.036 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:07:59 smithi187 ceph-mon[93536]: osdmap e834: 8 total, 8 up, 8 in 2024-03-29T05:08:00.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:59 smithi181 ceph-mon[101970]: failed to encode map e832 with expected crc 2024-03-29T05:08:00.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:59 smithi181 ceph-mon[101970]: failed to encode map e833 with expected crc 2024-03-29T05:08:00.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:59 smithi181 ceph-mon[101970]: failed to encode map e833 with expected crc 2024-03-29T05:08:00.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:07:59 smithi181 ceph-mon[101970]: osdmap e834: 8 total, 8 up, 8 in 2024-03-29T05:08:00.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:59 smithi181 ceph-mon[106466]: failed to encode map e832 with expected crc 2024-03-29T05:08:00.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:59 smithi181 ceph-mon[106466]: failed to encode map e833 with expected crc 2024-03-29T05:08:00.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:59 smithi181 ceph-mon[106466]: failed to encode map e833 with expected crc 2024-03-29T05:08:00.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:07:59 smithi181 ceph-mon[106466]: osdmap e834: 8 total, 8 up, 8 in 2024-03-29T05:08:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:00 smithi187 ceph-mon[93536]: failed to encode map e834 with expected crc 2024-03-29T05:08:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:00 smithi187 ceph-mon[93536]: pgmap v977: 329 pgs: 329 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:00 smithi187 ceph-mon[93536]: failed to encode map e833 with expected crc 2024-03-29T05:08:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:00 smithi187 ceph-mon[93536]: failed to encode map e834 with expected crc 2024-03-29T05:08:01.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:00 smithi187 ceph-mon[93536]: osdmap e835: 8 total, 8 up, 8 in 2024-03-29T05:08:01.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:00 smithi181 ceph-mon[101970]: failed to encode map e834 with expected crc 2024-03-29T05:08:01.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:00 smithi181 ceph-mon[101970]: pgmap v977: 329 pgs: 329 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:01.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:00 smithi181 ceph-mon[101970]: failed to encode map e833 with expected crc 2024-03-29T05:08:01.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:00 smithi181 ceph-mon[101970]: failed to encode map e834 with expected crc 2024-03-29T05:08:01.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:00 smithi181 ceph-mon[101970]: osdmap e835: 8 total, 8 up, 8 in 2024-03-29T05:08:01.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:00 smithi181 ceph-mon[106466]: failed to encode map e834 with expected crc 2024-03-29T05:08:01.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:00 smithi181 ceph-mon[106466]: pgmap v977: 329 pgs: 329 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:01.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:00 smithi181 ceph-mon[106466]: failed to encode map e833 with expected crc 2024-03-29T05:08:01.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:00 smithi181 ceph-mon[106466]: failed to encode map e834 with expected crc 2024-03-29T05:08:01.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:00 smithi181 ceph-mon[106466]: osdmap e835: 8 total, 8 up, 8 in 2024-03-29T05:08:02.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:02 smithi181 ceph-mon[101970]: osdmap e836: 8 total, 8 up, 8 in 2024-03-29T05:08:02.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:02 smithi181 ceph-mon[101970]: failed to encode map e835 with expected crc 2024-03-29T05:08:02.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:02 smithi181 ceph-mon[101970]: pgmap v980: 265 pgs: 265 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:02.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:02 smithi181 ceph-mon[101970]: failed to encode map e835 with expected crc 2024-03-29T05:08:02.955 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:02 smithi181 ceph-mon[101970]: failed to encode map e835 with expected crc 2024-03-29T05:08:02.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:02 smithi181 ceph-mon[106466]: osdmap e836: 8 total, 8 up, 8 in 2024-03-29T05:08:02.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:02 smithi181 ceph-mon[106466]: failed to encode map e835 with expected crc 2024-03-29T05:08:02.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:02 smithi181 ceph-mon[106466]: pgmap v980: 265 pgs: 265 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:02.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:02 smithi181 ceph-mon[106466]: failed to encode map e835 with expected crc 2024-03-29T05:08:02.956 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:02 smithi181 ceph-mon[106466]: failed to encode map e835 with expected crc 2024-03-29T05:08:03.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:02 smithi187 ceph-mon[93536]: osdmap e836: 8 total, 8 up, 8 in 2024-03-29T05:08:03.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:02 smithi187 ceph-mon[93536]: failed to encode map e835 with expected crc 2024-03-29T05:08:03.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:02 smithi187 ceph-mon[93536]: pgmap v980: 265 pgs: 265 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:03.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:02 smithi187 ceph-mon[93536]: failed to encode map e835 with expected crc 2024-03-29T05:08:03.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:02 smithi187 ceph-mon[93536]: failed to encode map e835 with expected crc 2024-03-29T05:08:04.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:03 smithi187 ceph-mon[93536]: failed to encode map e837 with expected crc 2024-03-29T05:08:04.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:03 smithi187 ceph-mon[93536]: osdmap e837: 8 total, 8 up, 8 in 2024-03-29T05:08:04.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:03 smithi187 ceph-mon[93536]: failed to encode map e837 with expected crc 2024-03-29T05:08:04.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:03 smithi187 ceph-mon[93536]: failed to encode map e837 with expected crc 2024-03-29T05:08:04.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:03 smithi181 ceph-mon[101970]: failed to encode map e837 with expected crc 2024-03-29T05:08:04.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:03 smithi181 ceph-mon[101970]: osdmap e837: 8 total, 8 up, 8 in 2024-03-29T05:08:04.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:03 smithi181 ceph-mon[101970]: failed to encode map e837 with expected crc 2024-03-29T05:08:04.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:03 smithi181 ceph-mon[101970]: failed to encode map e837 with expected crc 2024-03-29T05:08:04.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:03 smithi181 ceph-mon[106466]: failed to encode map e837 with expected crc 2024-03-29T05:08:04.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:03 smithi181 ceph-mon[106466]: osdmap e837: 8 total, 8 up, 8 in 2024-03-29T05:08:04.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:03 smithi181 ceph-mon[106466]: failed to encode map e837 with expected crc 2024-03-29T05:08:04.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:03 smithi181 ceph-mon[106466]: failed to encode map e837 with expected crc 2024-03-29T05:08:05.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:04 smithi187 ceph-mon[93536]: failed to encode map e838 with expected crc 2024-03-29T05:08:05.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:04 smithi187 ceph-mon[93536]: osdmap e838: 8 total, 8 up, 8 in 2024-03-29T05:08:05.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:04 smithi187 ceph-mon[93536]: pgmap v983: 201 pgs: 201 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:05.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:04 smithi187 ceph-mon[93536]: failed to encode map e838 with expected crc 2024-03-29T05:08:05.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:04 smithi187 ceph-mon[93536]: failed to encode map e838 with expected crc 2024-03-29T05:08:05.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:04 smithi181 ceph-mon[101970]: failed to encode map e838 with expected crc 2024-03-29T05:08:05.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:04 smithi181 ceph-mon[101970]: osdmap e838: 8 total, 8 up, 8 in 2024-03-29T05:08:05.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:04 smithi181 ceph-mon[101970]: pgmap v983: 201 pgs: 201 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:05.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:04 smithi181 ceph-mon[101970]: failed to encode map e838 with expected crc 2024-03-29T05:08:05.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:04 smithi181 ceph-mon[101970]: failed to encode map e838 with expected crc 2024-03-29T05:08:05.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:04 smithi181 ceph-mon[106466]: failed to encode map e838 with expected crc 2024-03-29T05:08:05.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:04 smithi181 ceph-mon[106466]: osdmap e838: 8 total, 8 up, 8 in 2024-03-29T05:08:05.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:04 smithi181 ceph-mon[106466]: pgmap v983: 201 pgs: 201 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:05.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:04 smithi181 ceph-mon[106466]: failed to encode map e838 with expected crc 2024-03-29T05:08:05.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:04 smithi181 ceph-mon[106466]: failed to encode map e838 with expected crc 2024-03-29T05:08:06.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:05 smithi187 ceph-mon[93536]: osdmap e839: 8 total, 8 up, 8 in 2024-03-29T05:08:06.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:05 smithi187 ceph-mon[93536]: failed to encode map e839 with expected crc 2024-03-29T05:08:06.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:05 smithi187 ceph-mon[93536]: failed to encode map e839 with expected crc 2024-03-29T05:08:06.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:05 smithi181 ceph-mon[101970]: osdmap e839: 8 total, 8 up, 8 in 2024-03-29T05:08:06.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:05 smithi181 ceph-mon[101970]: failed to encode map e839 with expected crc 2024-03-29T05:08:06.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:05 smithi181 ceph-mon[101970]: failed to encode map e839 with expected crc 2024-03-29T05:08:06.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:05 smithi181 ceph-mon[106466]: osdmap e839: 8 total, 8 up, 8 in 2024-03-29T05:08:06.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:05 smithi181 ceph-mon[106466]: failed to encode map e839 with expected crc 2024-03-29T05:08:06.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:05 smithi181 ceph-mon[106466]: failed to encode map e839 with expected crc 2024-03-29T05:08:07.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:06 smithi187 ceph-mon[93536]: failed to encode map e839 with expected crc 2024-03-29T05:08:07.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:06 smithi187 ceph-mon[93536]: pgmap v985: 169 pgs: 169 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:07.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:06 smithi187 ceph-mon[93536]: failed to encode map e840 with expected crc 2024-03-29T05:08:07.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:06 smithi187 ceph-mon[93536]: osdmap e840: 8 total, 8 up, 8 in 2024-03-29T05:08:07.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:06 smithi187 ceph-mon[93536]: failed to encode map e840 with expected crc 2024-03-29T05:08:07.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:06 smithi187 ceph-mon[93536]: failed to encode map e840 with expected crc 2024-03-29T05:08:07.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:06 smithi181 ceph-mon[101970]: failed to encode map e839 with expected crc 2024-03-29T05:08:07.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:06 smithi181 ceph-mon[101970]: pgmap v985: 169 pgs: 169 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:07.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:06 smithi181 ceph-mon[101970]: failed to encode map e840 with expected crc 2024-03-29T05:08:07.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:06 smithi181 ceph-mon[101970]: osdmap e840: 8 total, 8 up, 8 in 2024-03-29T05:08:07.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:06 smithi181 ceph-mon[101970]: failed to encode map e840 with expected crc 2024-03-29T05:08:07.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:06 smithi181 ceph-mon[101970]: failed to encode map e840 with expected crc 2024-03-29T05:08:07.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:06 smithi181 ceph-mon[106466]: failed to encode map e839 with expected crc 2024-03-29T05:08:07.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:06 smithi181 ceph-mon[106466]: pgmap v985: 169 pgs: 169 active+clean; 587 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:07.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:06 smithi181 ceph-mon[106466]: failed to encode map e840 with expected crc 2024-03-29T05:08:07.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:06 smithi181 ceph-mon[106466]: osdmap e840: 8 total, 8 up, 8 in 2024-03-29T05:08:07.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:06 smithi181 ceph-mon[106466]: failed to encode map e840 with expected crc 2024-03-29T05:08:07.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:06 smithi181 ceph-mon[106466]: failed to encode map e840 with expected crc 2024-03-29T05:08:07.748 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 159 tests from TestLibRBD (339571 ms total) 2024-03-29T05:08:07.748 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:08:07.748 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 15 tests from DiffIterateTest/0, where TypeParam = DiffIterateParams 2024-03-29T05:08:07.776 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/0.DiffIterate 2024-03-29T05:08:08.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:07 smithi187 ceph-mon[93536]: failed to encode map e841 with expected crc 2024-03-29T05:08:08.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:07 smithi187 ceph-mon[93536]: osdmap e841: 8 total, 8 up, 8 in 2024-03-29T05:08:08.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:07 smithi187 ceph-mon[93536]: failed to encode map e841 with expected crc 2024-03-29T05:08:08.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:07 smithi187 ceph-mon[93536]: failed to encode map e841 with expected crc 2024-03-29T05:08:08.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:07 smithi181 ceph-mon[106466]: failed to encode map e841 with expected crc 2024-03-29T05:08:08.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:07 smithi181 ceph-mon[106466]: osdmap e841: 8 total, 8 up, 8 in 2024-03-29T05:08:08.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:07 smithi181 ceph-mon[106466]: failed to encode map e841 with expected crc 2024-03-29T05:08:08.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:07 smithi181 ceph-mon[106466]: failed to encode map e841 with expected crc 2024-03-29T05:08:08.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:07 smithi181 ceph-mon[101970]: failed to encode map e841 with expected crc 2024-03-29T05:08:08.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:07 smithi181 ceph-mon[101970]: osdmap e841: 8 total, 8 up, 8 in 2024-03-29T05:08:08.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:07 smithi181 ceph-mon[101970]: failed to encode map e841 with expected crc 2024-03-29T05:08:08.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:07 smithi181 ceph-mon[101970]: failed to encode map e841 with expected crc 2024-03-29T05:08:09.106 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:08:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:08:08] "GET /metrics HTTP/1.1" 200 62906 "" "Prometheus/2.43.0" 2024-03-29T05:08:09.106 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:08 smithi181 ceph-mon[101970]: pgmap v988: 105 pgs: 105 active+clean; 586 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:09.106 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:08 smithi181 ceph-mon[101970]: failed to encode map e842 with expected crc 2024-03-29T05:08:09.106 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:08 smithi181 ceph-mon[101970]: osdmap e842: 8 total, 8 up, 8 in 2024-03-29T05:08:09.107 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:08 smithi181 ceph-mon[101970]: failed to encode map e842 with expected crc 2024-03-29T05:08:09.107 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:08 smithi181 ceph-mon[101970]: failed to encode map e842 with expected crc 2024-03-29T05:08:09.107 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:08 smithi181 ceph-mon[106466]: pgmap v988: 105 pgs: 105 active+clean; 586 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:09.107 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:08 smithi181 ceph-mon[106466]: failed to encode map e842 with expected crc 2024-03-29T05:08:09.107 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:08 smithi181 ceph-mon[106466]: osdmap e842: 8 total, 8 up, 8 in 2024-03-29T05:08:09.107 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:08 smithi181 ceph-mon[106466]: failed to encode map e842 with expected crc 2024-03-29T05:08:09.107 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:08 smithi181 ceph-mon[106466]: failed to encode map e842 with expected crc 2024-03-29T05:08:09.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:08 smithi187 ceph-mon[93536]: pgmap v988: 105 pgs: 105 active+clean; 586 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:09.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:08 smithi187 ceph-mon[93536]: failed to encode map e842 with expected crc 2024-03-29T05:08:09.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:08 smithi187 ceph-mon[93536]: osdmap e842: 8 total, 8 up, 8 in 2024-03-29T05:08:09.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:08 smithi187 ceph-mon[93536]: failed to encode map e842 with expected crc 2024-03-29T05:08:09.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:08 smithi187 ceph-mon[93536]: failed to encode map e842 with expected crc 2024-03-29T05:08:09.774 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:08:10.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:09 smithi187 ceph-mon[93536]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:09 smithi187 ceph-mon[93536]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:09 smithi187 ceph-mon[93536]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:09 smithi187 ceph-mon[93536]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:09 smithi187 ceph-mon[93536]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:09 smithi187 ceph-mon[93536]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:09 smithi187 ceph-mon[93536]: osdmap e843: 8 total, 8 up, 8 in 2024-03-29T05:08:10.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:09 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/414232504' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:08:10.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:09 smithi187 ceph-mon[93536]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:09 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:08:10.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[101970]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[101970]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[101970]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[101970]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[101970]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[101970]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[101970]: osdmap e843: 8 total, 8 up, 8 in 2024-03-29T05:08:10.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/414232504' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:08:10.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[101970]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:08:10.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[106466]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[106466]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[106466]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[106466]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[106466]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[106466]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[106466]: osdmap e843: 8 total, 8 up, 8 in 2024-03-29T05:08:10.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/414232504' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:08:10.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[106466]: failed to encode map e843 with expected crc 2024-03-29T05:08:10.206 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:09 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:08:11.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:10 smithi187 ceph-mon[93536]: pgmap v991: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:11.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:10 smithi187 ceph-mon[93536]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:08:11.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:10 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:08:11.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:10 smithi187 ceph-mon[93536]: failed to encode map e844 with expected crc 2024-03-29T05:08:11.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:10 smithi187 ceph-mon[93536]: osdmap e844: 8 total, 8 up, 8 in 2024-03-29T05:08:11.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:10 smithi187 ceph-mon[93536]: failed to encode map e844 with expected crc 2024-03-29T05:08:11.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:10 smithi187 ceph-mon[93536]: failed to encode map e844 with expected crc 2024-03-29T05:08:11.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:10 smithi187 ceph-mon[93536]: failed to encode map e844 with expected crc 2024-03-29T05:08:11.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:10 smithi181 ceph-mon[106466]: pgmap v991: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:11.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:10 smithi181 ceph-mon[106466]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:08:11.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:10 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:08:11.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:10 smithi181 ceph-mon[106466]: failed to encode map e844 with expected crc 2024-03-29T05:08:11.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:10 smithi181 ceph-mon[106466]: osdmap e844: 8 total, 8 up, 8 in 2024-03-29T05:08:11.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:10 smithi181 ceph-mon[106466]: failed to encode map e844 with expected crc 2024-03-29T05:08:11.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:10 smithi181 ceph-mon[106466]: failed to encode map e844 with expected crc 2024-03-29T05:08:11.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:10 smithi181 ceph-mon[106466]: failed to encode map e844 with expected crc 2024-03-29T05:08:11.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:10 smithi181 ceph-mon[101970]: pgmap v991: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:11.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:10 smithi181 ceph-mon[101970]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:08:11.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:10 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:08:11.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:10 smithi181 ceph-mon[101970]: failed to encode map e844 with expected crc 2024-03-29T05:08:11.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:10 smithi181 ceph-mon[101970]: osdmap e844: 8 total, 8 up, 8 in 2024-03-29T05:08:11.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:10 smithi181 ceph-mon[101970]: failed to encode map e844 with expected crc 2024-03-29T05:08:11.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:10 smithi181 ceph-mon[101970]: failed to encode map e844 with expected crc 2024-03-29T05:08:11.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:10 smithi181 ceph-mon[101970]: failed to encode map e844 with expected crc 2024-03-29T05:08:11.807 INFO:tasks.workunit.client.0.smithi181.stdout: wrote [89621~12946,993455~74783,3165167~33207,7334312~76306,8007948~33167,11989644~74514,13091539~51212,15252009~50512,15889888~63575,17994952~79537] 2024-03-29T05:08:12.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:11 smithi187 ceph-mon[93536]: failed to encode map e845 with expected crc 2024-03-29T05:08:12.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:11 smithi187 ceph-mon[93536]: osdmap e845: 8 total, 8 up, 8 in 2024-03-29T05:08:12.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:11 smithi187 ceph-mon[93536]: failed to encode map e845 with expected crc 2024-03-29T05:08:12.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:12.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:08:12.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:11 smithi181 ceph-mon[106466]: failed to encode map e845 with expected crc 2024-03-29T05:08:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:11 smithi181 ceph-mon[106466]: osdmap e845: 8 total, 8 up, 8 in 2024-03-29T05:08:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:11 smithi181 ceph-mon[106466]: failed to encode map e845 with expected crc 2024-03-29T05:08:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:12.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:08:12.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:11 smithi181 ceph-mon[101970]: failed to encode map e845 with expected crc 2024-03-29T05:08:12.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:11 smithi181 ceph-mon[101970]: osdmap e845: 8 total, 8 up, 8 in 2024-03-29T05:08:12.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:11 smithi181 ceph-mon[101970]: failed to encode map e845 with expected crc 2024-03-29T05:08:12.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:12.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:08:12.806 INFO:tasks.workunit.client.0.smithi181.stdout: wrote [1767679~59468,1901715~78955,4475233~12997,11800518~79657,12523007~50681,12919219~2354,17420572~101257,18006835~96124,20684669~32299] 2024-03-29T05:08:12.812 INFO:tasks.workunit.client.0.smithi181.stdout: diff was [1767679~59468,1901715~78955,4475233~12997,11800518~79657,12064158~509530,12919219~2354,17420572~101257,18006835~2710133] 2024-03-29T05:08:12.816 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/0.DiffIterate (5039 ms) 2024-03-29T05:08:12.816 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministic 2024-03-29T05:08:12.816 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:08:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:12 smithi187 ceph-mon[93536]: pgmap v994: 105 pgs: 15 unknown, 90 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:12 smithi187 ceph-mon[93536]: failed to encode map e846 with expected crc 2024-03-29T05:08:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:12 smithi187 ceph-mon[93536]: osdmap e846: 8 total, 8 up, 8 in 2024-03-29T05:08:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:12 smithi187 ceph-mon[93536]: failed to encode map e846 with expected crc 2024-03-29T05:08:13.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:12 smithi187 ceph-mon[93536]: failed to encode map e846 with expected crc 2024-03-29T05:08:13.136 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:12 smithi187 ceph-mon[93536]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:08:13.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:12 smithi181 ceph-mon[106466]: pgmap v994: 105 pgs: 15 unknown, 90 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:13.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:12 smithi181 ceph-mon[106466]: failed to encode map e846 with expected crc 2024-03-29T05:08:13.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:12 smithi181 ceph-mon[106466]: osdmap e846: 8 total, 8 up, 8 in 2024-03-29T05:08:13.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:12 smithi181 ceph-mon[106466]: failed to encode map e846 with expected crc 2024-03-29T05:08:13.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:12 smithi181 ceph-mon[106466]: failed to encode map e846 with expected crc 2024-03-29T05:08:13.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:12 smithi181 ceph-mon[106466]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:08:13.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:12 smithi181 ceph-mon[101970]: pgmap v994: 105 pgs: 15 unknown, 90 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:13.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:12 smithi181 ceph-mon[101970]: failed to encode map e846 with expected crc 2024-03-29T05:08:13.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:12 smithi181 ceph-mon[101970]: osdmap e846: 8 total, 8 up, 8 in 2024-03-29T05:08:13.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:12 smithi181 ceph-mon[101970]: failed to encode map e846 with expected crc 2024-03-29T05:08:13.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:12 smithi181 ceph-mon[101970]: failed to encode map e846 with expected crc 2024-03-29T05:08:13.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:12 smithi181 ceph-mon[101970]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T05:08:14.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:13 smithi187 ceph-mon[93536]: osdmap e847: 8 total, 8 up, 8 in 2024-03-29T05:08:14.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:13 smithi187 ceph-mon[93536]: failed to encode map e847 with expected crc 2024-03-29T05:08:14.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:13 smithi187 ceph-mon[93536]: failed to encode map e847 with expected crc 2024-03-29T05:08:14.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:13 smithi187 ceph-mon[93536]: failed to encode map e847 with expected crc 2024-03-29T05:08:14.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:13 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:08:14.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:13 smithi181 ceph-mon[101970]: osdmap e847: 8 total, 8 up, 8 in 2024-03-29T05:08:14.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:13 smithi181 ceph-mon[101970]: failed to encode map e847 with expected crc 2024-03-29T05:08:14.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:13 smithi181 ceph-mon[101970]: failed to encode map e847 with expected crc 2024-03-29T05:08:14.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:13 smithi181 ceph-mon[101970]: failed to encode map e847 with expected crc 2024-03-29T05:08:14.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:13 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:08:14.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:13 smithi181 ceph-mon[106466]: osdmap e847: 8 total, 8 up, 8 in 2024-03-29T05:08:14.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:13 smithi181 ceph-mon[106466]: failed to encode map e847 with expected crc 2024-03-29T05:08:14.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:13 smithi181 ceph-mon[106466]: failed to encode map e847 with expected crc 2024-03-29T05:08:14.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:13 smithi181 ceph-mon[106466]: failed to encode map e847 with expected crc 2024-03-29T05:08:14.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:13 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:08:14.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:14 smithi187 ceph-mon[93536]: pgmap v997: 105 pgs: 105 active+clean; 7.5 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 119 KiB/s wr, 4 op/s 2024-03-29T05:08:14.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:14 smithi187 ceph-mon[93536]: failed to encode map e848 with expected crc 2024-03-29T05:08:14.885 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:14 smithi187 ceph-mon[93536]: osdmap e848: 8 total, 8 up, 8 in 2024-03-29T05:08:14.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:14 smithi181 ceph-mon[101970]: pgmap v997: 105 pgs: 105 active+clean; 7.5 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 119 KiB/s wr, 4 op/s 2024-03-29T05:08:14.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:14 smithi181 ceph-mon[101970]: failed to encode map e848 with expected crc 2024-03-29T05:08:14.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:14 smithi181 ceph-mon[101970]: osdmap e848: 8 total, 8 up, 8 in 2024-03-29T05:08:14.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:14 smithi181 ceph-mon[106466]: pgmap v997: 105 pgs: 105 active+clean; 7.5 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 119 KiB/s wr, 4 op/s 2024-03-29T05:08:14.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:14 smithi181 ceph-mon[106466]: failed to encode map e848 with expected crc 2024-03-29T05:08:14.955 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:14 smithi181 ceph-mon[106466]: osdmap e848: 8 total, 8 up, 8 in 2024-03-29T05:08:15.839 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministic (3023 ms) 2024-03-29T05:08:15.839 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministicPP 2024-03-29T05:08:15.839 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:08:15.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:15 smithi181 ceph-mon[101970]: osdmap e849: 8 total, 8 up, 8 in 2024-03-29T05:08:15.954 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:15 smithi181 ceph-mon[101970]: failed to encode map e849 with expected crc 2024-03-29T05:08:15.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:15 smithi181 ceph-mon[106466]: osdmap e849: 8 total, 8 up, 8 in 2024-03-29T05:08:15.954 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:15 smithi181 ceph-mon[106466]: failed to encode map e849 with expected crc 2024-03-29T05:08:16.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:15 smithi187 ceph-mon[93536]: osdmap e849: 8 total, 8 up, 8 in 2024-03-29T05:08:16.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:15 smithi187 ceph-mon[93536]: failed to encode map e849 with expected crc 2024-03-29T05:08:17.078 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:16 smithi181 ceph-mon[101970]: pgmap v1000: 105 pgs: 105 active+clean; 7.5 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 119 KiB/s wr, 4 op/s 2024-03-29T05:08:17.078 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:16 smithi181 ceph-mon[101970]: failed to encode map e850 with expected crc 2024-03-29T05:08:17.078 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:16 smithi181 ceph-mon[101970]: osdmap e850: 8 total, 8 up, 8 in 2024-03-29T05:08:17.079 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:16 smithi181 ceph-mon[101970]: failed to encode map e850 with expected crc 2024-03-29T05:08:17.079 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:16 smithi181 ceph-mon[106466]: pgmap v1000: 105 pgs: 105 active+clean; 7.5 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 119 KiB/s wr, 4 op/s 2024-03-29T05:08:17.079 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:16 smithi181 ceph-mon[106466]: failed to encode map e850 with expected crc 2024-03-29T05:08:17.079 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:16 smithi181 ceph-mon[106466]: osdmap e850: 8 total, 8 up, 8 in 2024-03-29T05:08:17.079 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:16 smithi181 ceph-mon[106466]: failed to encode map e850 with expected crc 2024-03-29T05:08:17.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:16 smithi187 ceph-mon[93536]: pgmap v1000: 105 pgs: 105 active+clean; 7.5 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 119 KiB/s wr, 4 op/s 2024-03-29T05:08:17.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:16 smithi187 ceph-mon[93536]: failed to encode map e850 with expected crc 2024-03-29T05:08:17.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:16 smithi187 ceph-mon[93536]: osdmap e850: 8 total, 8 up, 8 in 2024-03-29T05:08:17.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:16 smithi187 ceph-mon[93536]: failed to encode map e850 with expected crc 2024-03-29T05:08:18.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:17 smithi187 ceph-mon[93536]: failed to encode map e851 with expected crc 2024-03-29T05:08:18.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:17 smithi187 ceph-mon[93536]: osdmap e851: 8 total, 8 up, 8 in 2024-03-29T05:08:18.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:17 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:18.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:17 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:18.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:17 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:18.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:17 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:18.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:17 smithi181 ceph-mon[101970]: failed to encode map e851 with expected crc 2024-03-29T05:08:18.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:17 smithi181 ceph-mon[101970]: osdmap e851: 8 total, 8 up, 8 in 2024-03-29T05:08:18.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:17 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:18.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:17 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:18.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:17 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:18.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:17 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:18.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:17 smithi181 ceph-mon[106466]: failed to encode map e851 with expected crc 2024-03-29T05:08:18.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:17 smithi181 ceph-mon[106466]: osdmap e851: 8 total, 8 up, 8 in 2024-03-29T05:08:18.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:17 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:18.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:17 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:18.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:17 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:18.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:17 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:18.856 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministicPP (3018 ms) 2024-03-29T05:08:18.856 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDiscard 2024-03-29T05:08:18.856 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:08:19.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:18 smithi187 ceph-mon[93536]: pgmap v1003: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 76 KiB/s rd, 170 KiB/s wr, 121 op/s 2024-03-29T05:08:19.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:18 smithi187 ceph-mon[93536]: osdmap e852: 8 total, 8 up, 8 in 2024-03-29T05:08:19.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:18 smithi187 ceph-mon[93536]: failed to encode map e852 with expected crc 2024-03-29T05:08:19.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:18 smithi187 ceph-mon[93536]: failed to encode map e852 with expected crc 2024-03-29T05:08:19.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:18 smithi187 ceph-mon[93536]: failed to encode map e852 with expected crc 2024-03-29T05:08:19.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:18 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:08:19.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:18 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:08:19.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:18 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:19.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:08:18] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2024-03-29T05:08:19.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-mon[106466]: pgmap v1003: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 76 KiB/s rd, 170 KiB/s wr, 121 op/s 2024-03-29T05:08:19.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-mon[106466]: osdmap e852: 8 total, 8 up, 8 in 2024-03-29T05:08:19.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-mon[106466]: failed to encode map e852 with expected crc 2024-03-29T05:08:19.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-mon[106466]: failed to encode map e852 with expected crc 2024-03-29T05:08:19.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-mon[106466]: failed to encode map e852 with expected crc 2024-03-29T05:08:19.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:08:19.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:08:19.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:19.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-mon[101970]: pgmap v1003: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 76 KiB/s rd, 170 KiB/s wr, 121 op/s 2024-03-29T05:08:19.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-mon[101970]: osdmap e852: 8 total, 8 up, 8 in 2024-03-29T05:08:19.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-mon[101970]: failed to encode map e852 with expected crc 2024-03-29T05:08:19.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-mon[101970]: failed to encode map e852 with expected crc 2024-03-29T05:08:19.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-mon[101970]: failed to encode map e852 with expected crc 2024-03-29T05:08:19.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:08:19.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:08:19.206 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:18 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:20.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:19 smithi187 ceph-mon[93536]: pgmap v1005: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 196 KiB/s wr, 139 op/s 2024-03-29T05:08:20.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:19 smithi187 ceph-mon[93536]: pgmap v1006: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 265 KiB/s wr, 189 op/s 2024-03-29T05:08:20.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:19 smithi187 ceph-mon[93536]: failed to encode map e853 with expected crc 2024-03-29T05:08:20.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:19 smithi187 ceph-mon[93536]: osdmap e853: 8 total, 8 up, 8 in 2024-03-29T05:08:20.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:19 smithi187 ceph-mon[93536]: failed to encode map e853 with expected crc 2024-03-29T05:08:20.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:19 smithi187 ceph-mon[93536]: failed to encode map e853 with expected crc 2024-03-29T05:08:20.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:19 smithi181 ceph-mon[106466]: pgmap v1005: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 196 KiB/s wr, 139 op/s 2024-03-29T05:08:20.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:19 smithi181 ceph-mon[106466]: pgmap v1006: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 265 KiB/s wr, 189 op/s 2024-03-29T05:08:20.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:19 smithi181 ceph-mon[106466]: failed to encode map e853 with expected crc 2024-03-29T05:08:20.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:19 smithi181 ceph-mon[106466]: osdmap e853: 8 total, 8 up, 8 in 2024-03-29T05:08:20.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:19 smithi181 ceph-mon[106466]: failed to encode map e853 with expected crc 2024-03-29T05:08:20.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:19 smithi181 ceph-mon[106466]: failed to encode map e853 with expected crc 2024-03-29T05:08:20.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:19 smithi181 ceph-mon[101970]: pgmap v1005: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 196 KiB/s wr, 139 op/s 2024-03-29T05:08:20.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:19 smithi181 ceph-mon[101970]: pgmap v1006: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 265 KiB/s wr, 189 op/s 2024-03-29T05:08:20.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:19 smithi181 ceph-mon[101970]: failed to encode map e853 with expected crc 2024-03-29T05:08:20.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:19 smithi181 ceph-mon[101970]: osdmap e853: 8 total, 8 up, 8 in 2024-03-29T05:08:20.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:19 smithi181 ceph-mon[101970]: failed to encode map e853 with expected crc 2024-03-29T05:08:20.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:19 smithi181 ceph-mon[101970]: failed to encode map e853 with expected crc 2024-03-29T05:08:21.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:20 smithi187 ceph-mon[93536]: failed to encode map e854 with expected crc 2024-03-29T05:08:21.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:20 smithi187 ceph-mon[93536]: osdmap e854: 8 total, 8 up, 8 in 2024-03-29T05:08:21.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:20 smithi187 ceph-mon[93536]: failed to encode map e854 with expected crc 2024-03-29T05:08:21.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:20 smithi187 ceph-mon[93536]: failed to encode map e854 with expected crc 2024-03-29T05:08:21.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:20 smithi181 ceph-mon[106466]: failed to encode map e854 with expected crc 2024-03-29T05:08:21.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:20 smithi181 ceph-mon[106466]: osdmap e854: 8 total, 8 up, 8 in 2024-03-29T05:08:21.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:20 smithi181 ceph-mon[106466]: failed to encode map e854 with expected crc 2024-03-29T05:08:21.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:20 smithi181 ceph-mon[106466]: failed to encode map e854 with expected crc 2024-03-29T05:08:21.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:20 smithi181 ceph-mon[101970]: failed to encode map e854 with expected crc 2024-03-29T05:08:21.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:20 smithi181 ceph-mon[101970]: osdmap e854: 8 total, 8 up, 8 in 2024-03-29T05:08:21.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:20 smithi181 ceph-mon[101970]: failed to encode map e854 with expected crc 2024-03-29T05:08:21.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:20 smithi181 ceph-mon[101970]: failed to encode map e854 with expected crc 2024-03-29T05:08:21.877 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/0.DiffIterateDiscard (3021 ms) 2024-03-29T05:08:21.877 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/0.DiffIterateTruncate 2024-03-29T05:08:21.877 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:08:22.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:21 smithi187 ceph-mon[93536]: pgmap v1009: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:22.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:21 smithi187 ceph-mon[93536]: failed to encode map e855 with expected crc 2024-03-29T05:08:22.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:21 smithi187 ceph-mon[93536]: osdmap e855: 8 total, 8 up, 8 in 2024-03-29T05:08:22.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:21 smithi187 ceph-mon[93536]: failed to encode map e855 with expected crc 2024-03-29T05:08:22.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:21 smithi187 ceph-mon[93536]: failed to encode map e855 with expected crc 2024-03-29T05:08:22.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:21 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:22.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:21 smithi181 ceph-mon[106466]: pgmap v1009: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:22.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:21 smithi181 ceph-mon[106466]: failed to encode map e855 with expected crc 2024-03-29T05:08:22.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:21 smithi181 ceph-mon[106466]: osdmap e855: 8 total, 8 up, 8 in 2024-03-29T05:08:22.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:21 smithi181 ceph-mon[106466]: failed to encode map e855 with expected crc 2024-03-29T05:08:22.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:21 smithi181 ceph-mon[106466]: failed to encode map e855 with expected crc 2024-03-29T05:08:22.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:21 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:22.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:21 smithi181 ceph-mon[101970]: pgmap v1009: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:08:22.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:21 smithi181 ceph-mon[101970]: failed to encode map e855 with expected crc 2024-03-29T05:08:22.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:21 smithi181 ceph-mon[101970]: osdmap e855: 8 total, 8 up, 8 in 2024-03-29T05:08:22.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:21 smithi181 ceph-mon[101970]: failed to encode map e855 with expected crc 2024-03-29T05:08:22.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:21 smithi181 ceph-mon[101970]: failed to encode map e855 with expected crc 2024-03-29T05:08:22.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:21 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:23.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:22 smithi187 ceph-mon[93536]: failed to encode map e856 with expected crc 2024-03-29T05:08:23.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:22 smithi187 ceph-mon[93536]: osdmap e856: 8 total, 8 up, 8 in 2024-03-29T05:08:23.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:22 smithi187 ceph-mon[93536]: failed to encode map e856 with expected crc 2024-03-29T05:08:23.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:22 smithi187 ceph-mon[93536]: failed to encode map e856 with expected crc 2024-03-29T05:08:23.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:22 smithi187 ceph-mon[93536]: osdmap e857: 8 total, 8 up, 8 in 2024-03-29T05:08:23.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:22 smithi181 ceph-mon[106466]: failed to encode map e856 with expected crc 2024-03-29T05:08:23.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:22 smithi181 ceph-mon[106466]: osdmap e856: 8 total, 8 up, 8 in 2024-03-29T05:08:23.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:22 smithi181 ceph-mon[106466]: failed to encode map e856 with expected crc 2024-03-29T05:08:23.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:22 smithi181 ceph-mon[106466]: failed to encode map e856 with expected crc 2024-03-29T05:08:23.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:22 smithi181 ceph-mon[106466]: osdmap e857: 8 total, 8 up, 8 in 2024-03-29T05:08:23.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:22 smithi181 ceph-mon[101970]: failed to encode map e856 with expected crc 2024-03-29T05:08:23.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:22 smithi181 ceph-mon[101970]: osdmap e856: 8 total, 8 up, 8 in 2024-03-29T05:08:23.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:22 smithi181 ceph-mon[101970]: failed to encode map e856 with expected crc 2024-03-29T05:08:23.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:22 smithi181 ceph-mon[101970]: failed to encode map e856 with expected crc 2024-03-29T05:08:23.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:22 smithi181 ceph-mon[101970]: osdmap e857: 8 total, 8 up, 8 in 2024-03-29T05:08:24.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:23 smithi187 ceph-mon[93536]: pgmap v1012: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 46 KiB/s rd, 12 KiB/s wr, 79 op/s 2024-03-29T05:08:24.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:23 smithi187 ceph-mon[93536]: failed to encode map e857 with expected crc 2024-03-29T05:08:24.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:23 smithi187 ceph-mon[93536]: failed to encode map e857 with expected crc 2024-03-29T05:08:24.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:23 smithi187 ceph-mon[93536]: osdmap e858: 8 total, 8 up, 8 in 2024-03-29T05:08:24.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:23 smithi181 ceph-mon[101970]: pgmap v1012: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 46 KiB/s rd, 12 KiB/s wr, 79 op/s 2024-03-29T05:08:24.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:23 smithi181 ceph-mon[101970]: failed to encode map e857 with expected crc 2024-03-29T05:08:24.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:23 smithi181 ceph-mon[101970]: failed to encode map e857 with expected crc 2024-03-29T05:08:24.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:23 smithi181 ceph-mon[101970]: osdmap e858: 8 total, 8 up, 8 in 2024-03-29T05:08:24.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:23 smithi181 ceph-mon[106466]: pgmap v1012: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 46 KiB/s rd, 12 KiB/s wr, 79 op/s 2024-03-29T05:08:24.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:23 smithi181 ceph-mon[106466]: failed to encode map e857 with expected crc 2024-03-29T05:08:24.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:23 smithi181 ceph-mon[106466]: failed to encode map e857 with expected crc 2024-03-29T05:08:24.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:23 smithi181 ceph-mon[106466]: osdmap e858: 8 total, 8 up, 8 in 2024-03-29T05:08:24.738 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/0.DiffIterateTruncate (2860 ms) 2024-03-29T05:08:24.738 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/0.DiffIterateWriteAndTruncate 2024-03-29T05:08:24.738 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:08:25.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:24 smithi187 ceph-mon[93536]: failed to encode map e858 with expected crc 2024-03-29T05:08:25.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:24 smithi187 ceph-mon[93536]: failed to encode map e858 with expected crc 2024-03-29T05:08:25.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:24 smithi187 ceph-mon[93536]: osdmap e859: 8 total, 8 up, 8 in 2024-03-29T05:08:25.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:24 smithi181 ceph-mon[106466]: failed to encode map e858 with expected crc 2024-03-29T05:08:25.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:24 smithi181 ceph-mon[106466]: failed to encode map e858 with expected crc 2024-03-29T05:08:25.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:24 smithi181 ceph-mon[106466]: osdmap e859: 8 total, 8 up, 8 in 2024-03-29T05:08:25.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:24 smithi181 ceph-mon[101970]: failed to encode map e858 with expected crc 2024-03-29T05:08:25.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:24 smithi181 ceph-mon[101970]: failed to encode map e858 with expected crc 2024-03-29T05:08:25.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:24 smithi181 ceph-mon[101970]: osdmap e859: 8 total, 8 up, 8 in 2024-03-29T05:08:26.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:25 smithi187 ceph-mon[93536]: pgmap v1015: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 76 KiB/s rd, 15 KiB/s wr, 120 op/s 2024-03-29T05:08:26.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:25 smithi187 ceph-mon[93536]: failed to encode map e859 with expected crc 2024-03-29T05:08:26.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:25 smithi187 ceph-mon[93536]: failed to encode map e859 with expected crc 2024-03-29T05:08:26.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:25 smithi187 ceph-mon[93536]: osdmap e860: 8 total, 8 up, 8 in 2024-03-29T05:08:26.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:25 smithi181 ceph-mon[106466]: pgmap v1015: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 76 KiB/s rd, 15 KiB/s wr, 120 op/s 2024-03-29T05:08:26.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:25 smithi181 ceph-mon[106466]: failed to encode map e859 with expected crc 2024-03-29T05:08:26.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:25 smithi181 ceph-mon[106466]: failed to encode map e859 with expected crc 2024-03-29T05:08:26.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:25 smithi181 ceph-mon[106466]: osdmap e860: 8 total, 8 up, 8 in 2024-03-29T05:08:26.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:25 smithi181 ceph-mon[101970]: pgmap v1015: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 76 KiB/s rd, 15 KiB/s wr, 120 op/s 2024-03-29T05:08:26.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:25 smithi181 ceph-mon[101970]: failed to encode map e859 with expected crc 2024-03-29T05:08:26.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:25 smithi181 ceph-mon[101970]: failed to encode map e859 with expected crc 2024-03-29T05:08:26.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:25 smithi181 ceph-mon[101970]: osdmap e860: 8 total, 8 up, 8 in 2024-03-29T05:08:27.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:26 smithi187 ceph-mon[93536]: failed to encode map e860 with expected crc 2024-03-29T05:08:27.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:26 smithi187 ceph-mon[93536]: failed to encode map e860 with expected crc 2024-03-29T05:08:27.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:26 smithi187 ceph-mon[93536]: failed to encode map e860 with expected crc 2024-03-29T05:08:27.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:27.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:08:27.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:26 smithi187 ceph-mon[93536]: osdmap e861: 8 total, 8 up, 8 in 2024-03-29T05:08:27.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:26 smithi181 ceph-mon[106466]: failed to encode map e860 with expected crc 2024-03-29T05:08:27.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:26 smithi181 ceph-mon[106466]: failed to encode map e860 with expected crc 2024-03-29T05:08:27.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:26 smithi181 ceph-mon[106466]: failed to encode map e860 with expected crc 2024-03-29T05:08:27.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:27.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:08:27.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:26 smithi181 ceph-mon[106466]: osdmap e861: 8 total, 8 up, 8 in 2024-03-29T05:08:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:26 smithi181 ceph-mon[101970]: failed to encode map e860 with expected crc 2024-03-29T05:08:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:26 smithi181 ceph-mon[101970]: failed to encode map e860 with expected crc 2024-03-29T05:08:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:26 smithi181 ceph-mon[101970]: failed to encode map e860 with expected crc 2024-03-29T05:08:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:08:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:08:27.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:26 smithi181 ceph-mon[101970]: osdmap e861: 8 total, 8 up, 8 in 2024-03-29T05:08:27.745 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/0.DiffIterateWriteAndTruncate (3007 ms) 2024-03-29T05:08:27.745 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStress 2024-03-29T05:08:27.745 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:08:27.826 INFO:tasks.workunit.client.0.smithi181.stdout: i=0 exists [4931332~6907042,18964428~5335267,44477769~6768688,78045251~6745665,162288778~2248162,179246198~7669011,277205940~3776570,352561988~6753589,404483307~1052296] wrote [4931332~6907042,18964428~5335267,44477769~6768688,78045251~6745665,162288778~2248162,179246198~7669011,277205940~3776570,352561988~6753589,404483307~1052296] 2024-03-29T05:08:28.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:27 smithi187 ceph-mon[93536]: pgmap v1018: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.5 KiB/s wr, 40 op/s 2024-03-29T05:08:28.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:27 smithi187 ceph-mon[93536]: failed to encode map e861 with expected crc 2024-03-29T05:08:28.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:27 smithi187 ceph-mon[93536]: failed to encode map e861 with expected crc 2024-03-29T05:08:28.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:27 smithi187 ceph-mon[93536]: failed to encode map e861 with expected crc 2024-03-29T05:08:28.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:27 smithi187 ceph-mon[93536]: failed to encode map e861 with expected crc 2024-03-29T05:08:28.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:27 smithi187 ceph-mon[93536]: osdmap e862: 8 total, 8 up, 8 in 2024-03-29T05:08:28.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:27 smithi181 ceph-mon[101970]: pgmap v1018: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.5 KiB/s wr, 40 op/s 2024-03-29T05:08:28.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:27 smithi181 ceph-mon[101970]: failed to encode map e861 with expected crc 2024-03-29T05:08:28.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:27 smithi181 ceph-mon[101970]: failed to encode map e861 with expected crc 2024-03-29T05:08:28.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:27 smithi181 ceph-mon[101970]: failed to encode map e861 with expected crc 2024-03-29T05:08:28.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:27 smithi181 ceph-mon[101970]: failed to encode map e861 with expected crc 2024-03-29T05:08:28.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:27 smithi181 ceph-mon[101970]: osdmap e862: 8 total, 8 up, 8 in 2024-03-29T05:08:28.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:27 smithi181 ceph-mon[106466]: pgmap v1018: 105 pgs: 105 active+clean; 19 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.5 KiB/s wr, 40 op/s 2024-03-29T05:08:28.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:27 smithi181 ceph-mon[106466]: failed to encode map e861 with expected crc 2024-03-29T05:08:28.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:27 smithi181 ceph-mon[106466]: failed to encode map e861 with expected crc 2024-03-29T05:08:28.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:27 smithi181 ceph-mon[106466]: failed to encode map e861 with expected crc 2024-03-29T05:08:28.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:27 smithi181 ceph-mon[106466]: failed to encode map e861 with expected crc 2024-03-29T05:08:28.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:27 smithi181 ceph-mon[106466]: osdmap e862: 8 total, 8 up, 8 in 2024-03-29T05:08:28.981 INFO:tasks.workunit.client.0.smithi181.stdout: i=1 exists [4931332~6907042,18964428~5335267,30979066~7452315,44477769~6768688,78045251~6745665,156036112~645685,162288778~2248162,179246198~7669011,197250192~5963915,242552251~2998060,276551166~4724153,350452960~8862617,370473318~7189972,404483307~1052296] wrote [30979066~7452315,79456923~3045340,156036112~645685,180137322~2342320,197250192~5963915,242552251~2998060,276551166~4724153,350452960~5753165,370473318~7189972] 2024-03-29T05:08:29.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:28 smithi187 ceph-mon[93536]: failed to encode map e862 with expected crc 2024-03-29T05:08:29.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:28 smithi187 ceph-mon[93536]: failed to encode map e862 with expected crc 2024-03-29T05:08:29.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:28 smithi187 ceph-mon[93536]: failed to encode map e862 with expected crc 2024-03-29T05:08:29.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:08:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:08:28] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.43.0" 2024-03-29T05:08:29.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:28 smithi181 ceph-mon[101970]: failed to encode map e862 with expected crc 2024-03-29T05:08:29.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:28 smithi181 ceph-mon[101970]: failed to encode map e862 with expected crc 2024-03-29T05:08:29.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:28 smithi181 ceph-mon[101970]: failed to encode map e862 with expected crc 2024-03-29T05:08:29.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:28 smithi181 ceph-mon[106466]: failed to encode map e862 with expected crc 2024-03-29T05:08:29.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:28 smithi181 ceph-mon[106466]: failed to encode map e862 with expected crc 2024-03-29T05:08:29.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:28 smithi181 ceph-mon[106466]: failed to encode map e862 with expected crc 2024-03-29T05:08:29.964 INFO:tasks.workunit.client.0.smithi181.stdout: i=2 exists [4931332~6907042,18964428~5335267,30979066~7452315,44477769~6768688,60821403~7408954,72033937~675494,78045251~6745665,112225849~4586789,156036112~645685,162288778~2248162,179246198~7669011,197250192~5963915,230584791~6738910,242552251~2998060,248873025~5038402,254279421~1816680,276551166~4724153,292525779~1466056,322822755~1665676,350452960~8862617,370473318~7189972,382386613~4399674,404483307~1052296] wrote [60821403~7408954,72033937~675494,112225849~4586789,230584791~6738910,248873025~5038402,254279421~1816680,292525779~1466056,322822755~1665676,382386613~4399674] 2024-03-29T05:08:30.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:29 smithi187 ceph-mon[93536]: pgmap v1021: 105 pgs: 105 active+clean; 21 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 515 KiB/s wr, 49 op/s 2024-03-29T05:08:30.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:29 smithi187 ceph-mon[93536]: failed to encode map e863 with expected crc 2024-03-29T05:08:30.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:29 smithi187 ceph-mon[93536]: osdmap e863: 8 total, 8 up, 8 in 2024-03-29T05:08:30.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:29 smithi187 ceph-mon[93536]: failed to encode map e863 with expected crc 2024-03-29T05:08:30.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:29 smithi187 ceph-mon[93536]: failed to encode map e863 with expected crc 2024-03-29T05:08:30.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:29 smithi181 ceph-mon[106466]: pgmap v1021: 105 pgs: 105 active+clean; 21 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 515 KiB/s wr, 49 op/s 2024-03-29T05:08:30.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:29 smithi181 ceph-mon[106466]: failed to encode map e863 with expected crc 2024-03-29T05:08:30.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:29 smithi181 ceph-mon[106466]: osdmap e863: 8 total, 8 up, 8 in 2024-03-29T05:08:30.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:29 smithi181 ceph-mon[106466]: failed to encode map e863 with expected crc 2024-03-29T05:08:30.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:29 smithi181 ceph-mon[106466]: failed to encode map e863 with expected crc 2024-03-29T05:08:30.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:29 smithi181 ceph-mon[101970]: pgmap v1021: 105 pgs: 105 active+clean; 21 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 515 KiB/s wr, 49 op/s 2024-03-29T05:08:30.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:29 smithi181 ceph-mon[101970]: failed to encode map e863 with expected crc 2024-03-29T05:08:30.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:29 smithi181 ceph-mon[101970]: osdmap e863: 8 total, 8 up, 8 in 2024-03-29T05:08:30.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:29 smithi181 ceph-mon[101970]: failed to encode map e863 with expected crc 2024-03-29T05:08:30.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:29 smithi181 ceph-mon[101970]: failed to encode map e863 with expected crc 2024-03-29T05:08:30.959 INFO:tasks.workunit.client.0.smithi181.stdout: i=3 exists [4931332~6907042,18964428~5335267,30979066~7452315,41252356~9994101,56829493~11400864,72033937~675494,78045251~6745665,112225849~4586789,156036112~645685,162288778~2248162,179246198~7669011,197250192~5963915,205369725~2447469,230584791~6738910,242552251~2998060,248873025~5038402,254279421~1816680,276551166~4724153,284493911~742072,292525779~1466056,322822755~1665676,329448555~32850,340503058~21662517,370473318~7189972,382386613~4399674,404483307~1052296,406124584~7978002] wrote [41252356~6856493,56829493~7192042,205369725~2447469,284493911~742072,329448555~32850,340503058~12114163,355236756~6928819,371174992~1402638,406124584~7978002] 2024-03-29T05:08:31.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:30 smithi181 ceph-mon[106466]: failed to encode map e864 with expected crc 2024-03-29T05:08:31.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:30 smithi181 ceph-mon[106466]: osdmap e864: 8 total, 8 up, 8 in 2024-03-29T05:08:31.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:30 smithi181 ceph-mon[106466]: failed to encode map e864 with expected crc 2024-03-29T05:08:31.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:30 smithi181 ceph-mon[106466]: failed to encode map e864 with expected crc 2024-03-29T05:08:31.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:30 smithi181 ceph-mon[101970]: failed to encode map e864 with expected crc 2024-03-29T05:08:31.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:30 smithi181 ceph-mon[101970]: osdmap e864: 8 total, 8 up, 8 in 2024-03-29T05:08:31.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:30 smithi181 ceph-mon[101970]: failed to encode map e864 with expected crc 2024-03-29T05:08:31.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:30 smithi181 ceph-mon[101970]: failed to encode map e864 with expected crc 2024-03-29T05:08:31.259 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:30 smithi187 ceph-mon[93536]: failed to encode map e864 with expected crc 2024-03-29T05:08:31.259 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:30 smithi187 ceph-mon[93536]: osdmap e864: 8 total, 8 up, 8 in 2024-03-29T05:08:31.259 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:30 smithi187 ceph-mon[93536]: failed to encode map e864 with expected crc 2024-03-29T05:08:31.259 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:30 smithi187 ceph-mon[93536]: failed to encode map e864 with expected crc 2024-03-29T05:08:32.014 INFO:tasks.workunit.client.0.smithi181.stdout: i=4 exists [4931332~6907042,18964428~5335267,30979066~7452315,41252356~9994101,56829493~11400864,72033937~675494,78045251~6745665,86382691~5152182,112225849~4586789,156036112~645685,162288778~2248162,167413531~1796768,173109571~13805638,197250192~5963915,205369725~2447469,230584791~9848925,242552251~2998060,248873025~5038402,254279421~1816680,276551166~7515111,284493911~742072,292525779~1466056,322822755~1665676,329448555~32850,340503058~21662517,369627346~8035944,382386613~4399674,404483307~1052296,406124584~7978002] wrote [31236423~4602591,66224838~1583868,79341797~5078800,86382691~5152182,167413531~1796768,173109571~9193179,232982965~7450751,276609835~7456442,369627346~5540348] 2024-03-29T05:08:32.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:31 smithi181 ceph-mon[106466]: pgmap v1024: 105 pgs: 105 active+clean; 43 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 5.6 MiB/s wr, 152 op/s 2024-03-29T05:08:32.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:31 smithi181 ceph-mon[106466]: failed to encode map e865 with expected crc 2024-03-29T05:08:32.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:31 smithi181 ceph-mon[106466]: osdmap e865: 8 total, 8 up, 8 in 2024-03-29T05:08:32.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:31 smithi181 ceph-mon[106466]: failed to encode map e865 with expected crc 2024-03-29T05:08:32.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:31 smithi181 ceph-mon[106466]: failed to encode map e865 with expected crc 2024-03-29T05:08:32.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:31 smithi181 ceph-mon[101970]: pgmap v1024: 105 pgs: 105 active+clean; 43 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 5.6 MiB/s wr, 152 op/s 2024-03-29T05:08:32.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:31 smithi181 ceph-mon[101970]: failed to encode map e865 with expected crc 2024-03-29T05:08:32.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:31 smithi181 ceph-mon[101970]: osdmap e865: 8 total, 8 up, 8 in 2024-03-29T05:08:32.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:31 smithi181 ceph-mon[101970]: failed to encode map e865 with expected crc 2024-03-29T05:08:32.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:31 smithi181 ceph-mon[101970]: failed to encode map e865 with expected crc 2024-03-29T05:08:32.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:31 smithi187 ceph-mon[93536]: pgmap v1024: 105 pgs: 105 active+clean; 43 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 5.6 MiB/s wr, 152 op/s 2024-03-29T05:08:32.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:31 smithi187 ceph-mon[93536]: failed to encode map e865 with expected crc 2024-03-29T05:08:32.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:31 smithi187 ceph-mon[93536]: osdmap e865: 8 total, 8 up, 8 in 2024-03-29T05:08:32.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:31 smithi187 ceph-mon[93536]: failed to encode map e865 with expected crc 2024-03-29T05:08:32.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:31 smithi187 ceph-mon[93536]: failed to encode map e865 with expected crc 2024-03-29T05:08:32.954 INFO:tasks.workunit.client.0.smithi181.stdout: i=5 exists [4931332~6907042,18964428~5335267,30979066~7452315,41252356~9994101,56829493~11400864,72033937~675494,78045251~6745665,86382691~5152182,111013211~305386,112225849~4586789,156036112~645685,161975827~3924913,167413531~1796768,173109571~13805638,197250192~5963915,205369725~2447469,214230671~974716,230584791~9848925,242552251~2998060,248873025~5038402,254279421~1816680,276551166~9603297,292525779~1466056,322822755~1665676,329448555~32850,340503058~26858463,369627346~8035944,378273435~8512852,403089332~1347239,404483307~1052296,406124584~7978002] wrote [44602410~5533264,111013211~305386,161975827~3924913,173301115~2201551,214230671~974716,282993096~3161367,293351984~1296,361795334~5566187,378273435~4517086,403089332~1347239] 2024-03-29T05:08:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:32 smithi181 ceph-mon[101970]: failed to encode map e866 with expected crc 2024-03-29T05:08:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:32 smithi181 ceph-mon[101970]: osdmap e866: 8 total, 8 up, 8 in 2024-03-29T05:08:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:32 smithi181 ceph-mon[101970]: failed to encode map e866 with expected crc 2024-03-29T05:08:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:32 smithi181 ceph-mon[101970]: failed to encode map e866 with expected crc 2024-03-29T05:08:33.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:32 smithi181 ceph-mon[101970]: failed to encode map e866 with expected crc 2024-03-29T05:08:33.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:32 smithi181 ceph-mon[106466]: failed to encode map e866 with expected crc 2024-03-29T05:08:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:32 smithi181 ceph-mon[106466]: osdmap e866: 8 total, 8 up, 8 in 2024-03-29T05:08:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:32 smithi181 ceph-mon[106466]: failed to encode map e866 with expected crc 2024-03-29T05:08:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:32 smithi181 ceph-mon[106466]: failed to encode map e866 with expected crc 2024-03-29T05:08:33.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:32 smithi181 ceph-mon[106466]: failed to encode map e866 with expected crc 2024-03-29T05:08:33.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:32 smithi187 ceph-mon[93536]: failed to encode map e866 with expected crc 2024-03-29T05:08:33.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:32 smithi187 ceph-mon[93536]: osdmap e866: 8 total, 8 up, 8 in 2024-03-29T05:08:33.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:32 smithi187 ceph-mon[93536]: failed to encode map e866 with expected crc 2024-03-29T05:08:33.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:32 smithi187 ceph-mon[93536]: failed to encode map e866 with expected crc 2024-03-29T05:08:33.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:32 smithi187 ceph-mon[93536]: failed to encode map e866 with expected crc 2024-03-29T05:08:33.975 INFO:tasks.workunit.client.0.smithi181.stdout: i=6 exists [404568~171768,2505262~9333112,18964428~5335267,30979066~7452315,41252356~9994101,56829493~11400864,72033937~675494,78045251~6745665,86382691~5152182,98583688~6003219,111013211~305386,112225849~4586789,139874896~9256679,156036112~645685,161975827~3924913,167413531~1796768,173109571~13805638,197250192~5963915,205369725~2447469,214230671~974716,230584791~9848925,242552251~2998060,247509107~8586994,276551166~9603297,292525779~1466056,317479958~508825,322822755~1665676,329448555~32850,340503058~26858463,369627346~8035944,378273435~8512852,403089332~1347239,404483307~1052296,406124584~7978002] wrote [404568~171768,2505262~8011646,98583688~6003219,139874896~9256679,247509107~7937298,276882371~6602782,317479958~508825,351472632~1404005,353208019~2502943] 2024-03-29T05:08:34.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:33 smithi181 ceph-mon[106466]: pgmap v1027: 105 pgs: 105 active+clean; 211 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 71 KiB/s rd, 36 MiB/s wr, 130 op/s 2024-03-29T05:08:34.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:33 smithi181 ceph-mon[106466]: osdmap e867: 8 total, 8 up, 8 in 2024-03-29T05:08:34.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:33 smithi181 ceph-mon[106466]: failed to encode map e867 with expected crc 2024-03-29T05:08:34.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:33 smithi181 ceph-mon[106466]: failed to encode map e867 with expected crc 2024-03-29T05:08:34.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:33 smithi181 ceph-mon[101970]: pgmap v1027: 105 pgs: 105 active+clean; 211 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 71 KiB/s rd, 36 MiB/s wr, 130 op/s 2024-03-29T05:08:34.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:33 smithi181 ceph-mon[101970]: osdmap e867: 8 total, 8 up, 8 in 2024-03-29T05:08:34.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:33 smithi181 ceph-mon[101970]: failed to encode map e867 with expected crc 2024-03-29T05:08:34.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:33 smithi181 ceph-mon[101970]: failed to encode map e867 with expected crc 2024-03-29T05:08:34.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:33 smithi187 ceph-mon[93536]: pgmap v1027: 105 pgs: 105 active+clean; 211 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 71 KiB/s rd, 36 MiB/s wr, 130 op/s 2024-03-29T05:08:34.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:33 smithi187 ceph-mon[93536]: osdmap e867: 8 total, 8 up, 8 in 2024-03-29T05:08:34.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:33 smithi187 ceph-mon[93536]: failed to encode map e867 with expected crc 2024-03-29T05:08:34.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:33 smithi187 ceph-mon[93536]: failed to encode map e867 with expected crc 2024-03-29T05:08:35.039 INFO:tasks.workunit.client.0.smithi181.stdout: i=7 exists [404568~171768,2505262~9333112,18964428~5335267,30979066~7452315,41252356~9994101,56829493~11400864,72033937~675494,74221867~1620152,78045251~6745665,86382691~5152182,98583688~6003219,111013211~305386,112225849~4586789,139874896~9256679,156036112~645685,161975827~3924913,167413531~1796768,173109571~23076369,197250192~11609484,214230671~974716,230584791~9848925,242552251~2998060,247509107~10202908,276551166~9603297,292525779~1466056,312283221~8455627,322822755~1665676,329448555~32850,340503058~26858463,369627346~8035944,378273435~8512852,403089332~1347239,404483307~1052296,406124584~7978002] wrote [42925201~4901136,57010643~8422112,74221867~1620152,186720600~9465340,200757268~8102408,253041056~4670959,312283221~8455627] 2024-03-29T05:08:35.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:34 smithi181 ceph-mon[101970]: failed to encode map e868 with expected crc 2024-03-29T05:08:35.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:34 smithi181 ceph-mon[101970]: osdmap e868: 8 total, 8 up, 8 in 2024-03-29T05:08:35.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:34 smithi181 ceph-mon[101970]: failed to encode map e868 with expected crc 2024-03-29T05:08:35.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:34 smithi181 ceph-mon[101970]: failed to encode map e868 with expected crc 2024-03-29T05:08:35.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:34 smithi181 ceph-mon[106466]: failed to encode map e868 with expected crc 2024-03-29T05:08:35.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:34 smithi181 ceph-mon[106466]: osdmap e868: 8 total, 8 up, 8 in 2024-03-29T05:08:35.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:34 smithi181 ceph-mon[106466]: failed to encode map e868 with expected crc 2024-03-29T05:08:35.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:34 smithi181 ceph-mon[106466]: failed to encode map e868 with expected crc 2024-03-29T05:08:35.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:34 smithi187 ceph-mon[93536]: failed to encode map e868 with expected crc 2024-03-29T05:08:35.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:34 smithi187 ceph-mon[93536]: osdmap e868: 8 total, 8 up, 8 in 2024-03-29T05:08:35.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:34 smithi187 ceph-mon[93536]: failed to encode map e868 with expected crc 2024-03-29T05:08:35.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:34 smithi187 ceph-mon[93536]: failed to encode map e868 with expected crc 2024-03-29T05:08:35.999 INFO:tasks.workunit.client.0.smithi181.stdout: i=8 exists [404568~171768,2505262~9333112,13801673~10498022,30979066~7452315,41252356~10121051,56829493~11452450,72033937~675494,74221867~1620152,78045251~6745665,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,112225849~4586789,124550033~2194459,139874896~9256679,156036112~645685,161975827~3924913,167413531~1796768,173109571~23076369,197250192~11609484,214230671~974716,230584791~9848925,242552251~2998060,246836530~10875485,276551166~9603297,292525779~1466056,312283221~8455627,322822755~1665676,329448555~32850,340503058~26858463,369627346~8035944,378273435~8512852,401507631~17271294] wrote [13801673~6528238,47224125~4149282,65183907~3098036,106697534~1611602,124550033~2194459,178154435~2311458,180963130~7391677,246836530~4250329,401507631~5667119,411204045~7574880] 2024-03-29T05:08:36.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:35 smithi181 ceph-mon[106466]: pgmap v1030: 105 pgs: 105 active+clean; 276 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 169 KiB/s rd, 45 MiB/s wr, 287 op/s 2024-03-29T05:08:36.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:35 smithi181 ceph-mon[106466]: osdmap e869: 8 total, 8 up, 8 in 2024-03-29T05:08:36.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:35 smithi181 ceph-mon[106466]: failed to encode map e869 with expected crc 2024-03-29T05:08:36.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:35 smithi181 ceph-mon[106466]: failed to encode map e869 with expected crc 2024-03-29T05:08:36.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:35 smithi181 ceph-mon[106466]: failed to encode map e869 with expected crc 2024-03-29T05:08:36.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:35 smithi181 ceph-mon[101970]: pgmap v1030: 105 pgs: 105 active+clean; 276 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 169 KiB/s rd, 45 MiB/s wr, 287 op/s 2024-03-29T05:08:36.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:35 smithi181 ceph-mon[101970]: osdmap e869: 8 total, 8 up, 8 in 2024-03-29T05:08:36.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:35 smithi181 ceph-mon[101970]: failed to encode map e869 with expected crc 2024-03-29T05:08:36.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:35 smithi181 ceph-mon[101970]: failed to encode map e869 with expected crc 2024-03-29T05:08:36.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:35 smithi181 ceph-mon[101970]: failed to encode map e869 with expected crc 2024-03-29T05:08:36.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:35 smithi187 ceph-mon[93536]: pgmap v1030: 105 pgs: 105 active+clean; 276 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 169 KiB/s rd, 45 MiB/s wr, 287 op/s 2024-03-29T05:08:36.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:35 smithi187 ceph-mon[93536]: osdmap e869: 8 total, 8 up, 8 in 2024-03-29T05:08:36.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:35 smithi187 ceph-mon[93536]: failed to encode map e869 with expected crc 2024-03-29T05:08:36.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:35 smithi187 ceph-mon[93536]: failed to encode map e869 with expected crc 2024-03-29T05:08:36.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:35 smithi187 ceph-mon[93536]: failed to encode map e869 with expected crc 2024-03-29T05:08:37.001 INFO:tasks.workunit.client.0.smithi181.stdout: i=9 exists [404568~171768,2505262~9333112,13801673~10498022,30979066~7452315,39741677~11631730,56829493~11452450,72033937~675494,74221867~1620152,78045251~6745665,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,112225849~4586789,124550033~2194459,139874896~9256679,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,214230671~974716,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,276551166~9603297,292525779~1466056,312283221~8455627,322822755~1665676,329448555~32850,340503058~26858463,369627346~8035944,378273435~10682903,397672230~21106695] wrote [39741677~6065998,47096226~1496609,143026722~1320660,192736931~5856241,207668912~4004045,263765730~2028814,382357326~6599012,397672230~6203578] 2024-03-29T05:08:37.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:36 smithi181 ceph-mon[106466]: osdmap e870: 8 total, 8 up, 8 in 2024-03-29T05:08:37.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:36 smithi181 ceph-mon[106466]: failed to encode map e870 with expected crc 2024-03-29T05:08:37.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:36 smithi181 ceph-mon[106466]: failed to encode map e870 with expected crc 2024-03-29T05:08:37.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:36 smithi181 ceph-mon[106466]: failed to encode map e870 with expected crc 2024-03-29T05:08:37.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:36 smithi181 ceph-mon[106466]: failed to encode map e870 with expected crc 2024-03-29T05:08:37.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:36 smithi181 ceph-mon[101970]: osdmap e870: 8 total, 8 up, 8 in 2024-03-29T05:08:37.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:36 smithi181 ceph-mon[101970]: failed to encode map e870 with expected crc 2024-03-29T05:08:37.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:36 smithi181 ceph-mon[101970]: failed to encode map e870 with expected crc 2024-03-29T05:08:37.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:36 smithi181 ceph-mon[101970]: failed to encode map e870 with expected crc 2024-03-29T05:08:37.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:36 smithi181 ceph-mon[101970]: failed to encode map e870 with expected crc 2024-03-29T05:08:37.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:36 smithi187 ceph-mon[93536]: osdmap e870: 8 total, 8 up, 8 in 2024-03-29T05:08:37.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:36 smithi187 ceph-mon[93536]: failed to encode map e870 with expected crc 2024-03-29T05:08:37.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:36 smithi187 ceph-mon[93536]: failed to encode map e870 with expected crc 2024-03-29T05:08:37.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:36 smithi187 ceph-mon[93536]: failed to encode map e870 with expected crc 2024-03-29T05:08:37.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:36 smithi187 ceph-mon[93536]: failed to encode map e870 with expected crc 2024-03-29T05:08:38.006 INFO:tasks.workunit.client.0.smithi181.stdout: i=10 exists [404568~171768,2505262~9333112,13801673~10498022,30979066~7452315,39741677~11631730,56829493~11452450,72033937~3808082,78045251~6745665,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~9256679,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,214230671~974716,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,276551166~9603297,292525779~1466056,295753178~3458866,312283221~8455627,322680407~1808024,329448555~32850,340503058~26858463,369311906~8351384,378273435~10682903,397672230~21106695] wrote [34926731~2450064,45179185~4805182,72073222~2361176,81352804~3099543,111888563~6642772,200988779~4276349,295753178~3458866,322680407~1513294,369311906~8107396,407807916~5424201] 2024-03-29T05:08:38.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:37 smithi181 ceph-mon[101970]: pgmap v1033: 105 pgs: 105 active+clean; 276 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 166 KiB/s rd, 14 MiB/s wr, 260 op/s 2024-03-29T05:08:38.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:37 smithi181 ceph-mon[101970]: failed to encode map e871 with expected crc 2024-03-29T05:08:38.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:37 smithi181 ceph-mon[101970]: osdmap e871: 8 total, 8 up, 8 in 2024-03-29T05:08:38.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:37 smithi181 ceph-mon[101970]: failed to encode map e871 with expected crc 2024-03-29T05:08:38.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:37 smithi181 ceph-mon[101970]: failed to encode map e871 with expected crc 2024-03-29T05:08:38.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:37 smithi181 ceph-mon[106466]: pgmap v1033: 105 pgs: 105 active+clean; 276 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 166 KiB/s rd, 14 MiB/s wr, 260 op/s 2024-03-29T05:08:38.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:37 smithi181 ceph-mon[106466]: failed to encode map e871 with expected crc 2024-03-29T05:08:38.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:37 smithi181 ceph-mon[106466]: osdmap e871: 8 total, 8 up, 8 in 2024-03-29T05:08:38.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:37 smithi181 ceph-mon[106466]: failed to encode map e871 with expected crc 2024-03-29T05:08:38.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:37 smithi181 ceph-mon[106466]: failed to encode map e871 with expected crc 2024-03-29T05:08:38.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:37 smithi187 ceph-mon[93536]: pgmap v1033: 105 pgs: 105 active+clean; 276 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 166 KiB/s rd, 14 MiB/s wr, 260 op/s 2024-03-29T05:08:38.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:37 smithi187 ceph-mon[93536]: failed to encode map e871 with expected crc 2024-03-29T05:08:38.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:37 smithi187 ceph-mon[93536]: osdmap e871: 8 total, 8 up, 8 in 2024-03-29T05:08:38.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:37 smithi187 ceph-mon[93536]: failed to encode map e871 with expected crc 2024-03-29T05:08:38.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:37 smithi187 ceph-mon[93536]: failed to encode map e871 with expected crc 2024-03-29T05:08:39.016 INFO:tasks.workunit.client.0.smithi181.stdout: i=11 exists [404568~171768,2505262~9333112,13801673~10498022,30979066~7452315,39741677~11631730,56829493~11452450,72033937~3808082,78045251~6745665,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,276551166~9603297,292525779~1466056,295753178~3458866,312283221~8455627,322513851~1989805,329343811~2701118,340147706~27213815,369311906~8351384,378273435~16507122,397672230~21106695] wrote [59139709~4018447,147439418~4104761,189795676~6858786,213342593~7281743,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,383035237~1558455,387938464~6842093] 2024-03-29T05:08:39.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:08:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:08:38] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.43.0" 2024-03-29T05:08:39.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:38 smithi181 ceph-mon[106466]: osdmap e872: 8 total, 8 up, 8 in 2024-03-29T05:08:39.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:38 smithi181 ceph-mon[106466]: failed to encode map e872 with expected crc 2024-03-29T05:08:39.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:38 smithi181 ceph-mon[106466]: failed to encode map e872 with expected crc 2024-03-29T05:08:39.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:38 smithi181 ceph-mon[106466]: failed to encode map e872 with expected crc 2024-03-29T05:08:39.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:38 smithi181 ceph-mon[106466]: failed to encode map e872 with expected crc 2024-03-29T05:08:39.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:38 smithi181 ceph-mon[101970]: osdmap e872: 8 total, 8 up, 8 in 2024-03-29T05:08:39.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:38 smithi181 ceph-mon[101970]: failed to encode map e872 with expected crc 2024-03-29T05:08:39.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:38 smithi181 ceph-mon[101970]: failed to encode map e872 with expected crc 2024-03-29T05:08:39.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:38 smithi181 ceph-mon[101970]: failed to encode map e872 with expected crc 2024-03-29T05:08:39.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:38 smithi181 ceph-mon[101970]: failed to encode map e872 with expected crc 2024-03-29T05:08:39.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:38 smithi187 ceph-mon[93536]: osdmap e872: 8 total, 8 up, 8 in 2024-03-29T05:08:39.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:38 smithi187 ceph-mon[93536]: failed to encode map e872 with expected crc 2024-03-29T05:08:39.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:38 smithi187 ceph-mon[93536]: failed to encode map e872 with expected crc 2024-03-29T05:08:39.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:38 smithi187 ceph-mon[93536]: failed to encode map e872 with expected crc 2024-03-29T05:08:39.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:38 smithi187 ceph-mon[93536]: failed to encode map e872 with expected crc 2024-03-29T05:08:40.019 INFO:tasks.workunit.client.0.smithi181.stdout: i=12 exists [404568~171768,2505262~9333112,13801673~10498022,28709632~1902975,30979066~7452315,39741677~11631730,56829493~11452450,72033937~3808082,78045251~6745665,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,340147706~27213815,369311906~8351384,378273435~16507122,397672230~21106695] wrote [28709632~1902975,78839889~3325874,183855054~107719,195956946~4329407,254724310~2114882,274173672~5897142,307919109~9983177] 2024-03-29T05:08:40.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:39 smithi181 ceph-mon[106466]: pgmap v1036: 105 pgs: 105 active+clean; 437 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 36 MiB/s wr, 19 op/s 2024-03-29T05:08:40.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:39 smithi181 ceph-mon[106466]: failed to encode map e873 with expected crc 2024-03-29T05:08:40.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:39 smithi181 ceph-mon[106466]: osdmap e873: 8 total, 8 up, 8 in 2024-03-29T05:08:40.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:39 smithi181 ceph-mon[106466]: failed to encode map e873 with expected crc 2024-03-29T05:08:40.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:39 smithi181 ceph-mon[106466]: failed to encode map e873 with expected crc 2024-03-29T05:08:40.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:39 smithi181 ceph-mon[101970]: pgmap v1036: 105 pgs: 105 active+clean; 437 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 36 MiB/s wr, 19 op/s 2024-03-29T05:08:40.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:39 smithi181 ceph-mon[101970]: failed to encode map e873 with expected crc 2024-03-29T05:08:40.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:39 smithi181 ceph-mon[101970]: osdmap e873: 8 total, 8 up, 8 in 2024-03-29T05:08:40.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:39 smithi181 ceph-mon[101970]: failed to encode map e873 with expected crc 2024-03-29T05:08:40.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:39 smithi181 ceph-mon[101970]: failed to encode map e873 with expected crc 2024-03-29T05:08:40.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:39 smithi187 ceph-mon[93536]: pgmap v1036: 105 pgs: 105 active+clean; 437 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 36 MiB/s wr, 19 op/s 2024-03-29T05:08:40.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:39 smithi187 ceph-mon[93536]: failed to encode map e873 with expected crc 2024-03-29T05:08:40.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:39 smithi187 ceph-mon[93536]: osdmap e873: 8 total, 8 up, 8 in 2024-03-29T05:08:40.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:39 smithi187 ceph-mon[93536]: failed to encode map e873 with expected crc 2024-03-29T05:08:40.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:39 smithi187 ceph-mon[93536]: failed to encode map e873 with expected crc 2024-03-29T05:08:41.053 INFO:tasks.workunit.client.0.smithi181.stdout: i=13 exists [404568~171768,2505262~9333112,13801673~10588900,28709632~1902975,30979066~7452315,39741677~11631730,56829493~11697280,72033937~3808082,78045251~6745665,86382691~5152182,98583688~6003219,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~40863094,378273435~40505490] wrote [18311766~6078807,57640836~6971573,66741740~1785033,109482434~4861457,127176832~7981584,183940397~5185711,336800196~6585868,365857476~4405229,394051856~4008344] 2024-03-29T05:08:41.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:40 smithi187 ceph-mon[93536]: failed to encode map e874 with expected crc 2024-03-29T05:08:41.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:40 smithi187 ceph-mon[93536]: osdmap e874: 8 total, 8 up, 8 in 2024-03-29T05:08:41.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:40 smithi187 ceph-mon[93536]: failed to encode map e874 with expected crc 2024-03-29T05:08:41.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:40 smithi187 ceph-mon[93536]: failed to encode map e874 with expected crc 2024-03-29T05:08:41.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:40 smithi187 ceph-mon[93536]: failed to encode map e874 with expected crc 2024-03-29T05:08:41.135 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:40 smithi187 ceph-mon[93536]: failed to encode map e874 with expected crc 2024-03-29T05:08:41.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:40 smithi181 ceph-mon[106466]: failed to encode map e874 with expected crc 2024-03-29T05:08:41.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:40 smithi181 ceph-mon[106466]: osdmap e874: 8 total, 8 up, 8 in 2024-03-29T05:08:41.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:40 smithi181 ceph-mon[106466]: failed to encode map e874 with expected crc 2024-03-29T05:08:41.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:40 smithi181 ceph-mon[106466]: failed to encode map e874 with expected crc 2024-03-29T05:08:41.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:40 smithi181 ceph-mon[106466]: failed to encode map e874 with expected crc 2024-03-29T05:08:41.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:40 smithi181 ceph-mon[106466]: failed to encode map e874 with expected crc 2024-03-29T05:08:41.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:40 smithi181 ceph-mon[101970]: failed to encode map e874 with expected crc 2024-03-29T05:08:41.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:40 smithi181 ceph-mon[101970]: osdmap e874: 8 total, 8 up, 8 in 2024-03-29T05:08:41.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:40 smithi181 ceph-mon[101970]: failed to encode map e874 with expected crc 2024-03-29T05:08:41.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:40 smithi181 ceph-mon[101970]: failed to encode map e874 with expected crc 2024-03-29T05:08:41.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:40 smithi181 ceph-mon[101970]: failed to encode map e874 with expected crc 2024-03-29T05:08:41.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:40 smithi181 ceph-mon[101970]: failed to encode map e874 with expected crc 2024-03-29T05:08:42.072 INFO:tasks.workunit.client.0.smithi181.stdout: i=14 exists [404568~171768,2505262~9333112,13801673~10588900,27077192~11354189,39741677~11631730,56829493~11697280,72033937~3808082,78045251~6745665,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~81978729] wrote [27077192~6988326,59746300~3583961,93785211~7481883,143272389~3772807,198059281~4283457,337214045~3795808,349983031~6849925,375001040~3381344,399456201~6910432] 2024-03-29T05:08:42.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:41 smithi187 ceph-mon[93536]: pgmap v1039: 105 pgs: 105 active+clean; 494 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 189 KiB/s rd, 51 MiB/s wr, 306 op/s 2024-03-29T05:08:42.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:41 smithi187 ceph-mon[93536]: failed to encode map e875 with expected crc 2024-03-29T05:08:42.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:41 smithi187 ceph-mon[93536]: osdmap e875: 8 total, 8 up, 8 in 2024-03-29T05:08:42.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:41 smithi187 ceph-mon[93536]: failed to encode map e875 with expected crc 2024-03-29T05:08:42.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:41 smithi187 ceph-mon[93536]: failed to encode map e875 with expected crc 2024-03-29T05:08:42.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:08:42.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:41 smithi181 ceph-mon[101970]: pgmap v1039: 105 pgs: 105 active+clean; 494 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 189 KiB/s rd, 51 MiB/s wr, 306 op/s 2024-03-29T05:08:42.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:41 smithi181 ceph-mon[101970]: failed to encode map e875 with expected crc 2024-03-29T05:08:42.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:41 smithi181 ceph-mon[101970]: osdmap e875: 8 total, 8 up, 8 in 2024-03-29T05:08:42.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:41 smithi181 ceph-mon[101970]: failed to encode map e875 with expected crc 2024-03-29T05:08:42.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:41 smithi181 ceph-mon[101970]: failed to encode map e875 with expected crc 2024-03-29T05:08:42.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:08:42.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:41 smithi181 ceph-mon[106466]: pgmap v1039: 105 pgs: 105 active+clean; 494 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 189 KiB/s rd, 51 MiB/s wr, 306 op/s 2024-03-29T05:08:42.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:41 smithi181 ceph-mon[106466]: failed to encode map e875 with expected crc 2024-03-29T05:08:42.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:41 smithi181 ceph-mon[106466]: osdmap e875: 8 total, 8 up, 8 in 2024-03-29T05:08:42.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:41 smithi181 ceph-mon[106466]: failed to encode map e875 with expected crc 2024-03-29T05:08:42.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:41 smithi181 ceph-mon[106466]: failed to encode map e875 with expected crc 2024-03-29T05:08:42.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:08:43.051 INFO:tasks.workunit.client.0.smithi181.stdout: i=15 exists [404568~171768,607349~11231025,13801673~10588900,27077192~11354189,39741677~12253486,56829493~11697280,72033937~3808082,78045251~6745665,84964601~734981,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~81978729] wrote [607349~4280051,33264680~4209092,50354796~1640367,84964601~734981,99411789~4934367,202433993~3601358,212065768~7948638,313401620~414791,381994856~6144098] 2024-03-29T05:08:43.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:42 smithi187 ceph-mon[93536]: osdmap e876: 8 total, 8 up, 8 in 2024-03-29T05:08:43.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:42 smithi187 ceph-mon[93536]: failed to encode map e876 with expected crc 2024-03-29T05:08:43.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:42 smithi187 ceph-mon[93536]: failed to encode map e876 with expected crc 2024-03-29T05:08:43.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:42 smithi187 ceph-mon[93536]: failed to encode map e876 with expected crc 2024-03-29T05:08:43.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:42 smithi181 ceph-mon[106466]: osdmap e876: 8 total, 8 up, 8 in 2024-03-29T05:08:43.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:42 smithi181 ceph-mon[106466]: failed to encode map e876 with expected crc 2024-03-29T05:08:43.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:42 smithi181 ceph-mon[106466]: failed to encode map e876 with expected crc 2024-03-29T05:08:43.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:42 smithi181 ceph-mon[106466]: failed to encode map e876 with expected crc 2024-03-29T05:08:43.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:42 smithi181 ceph-mon[101970]: osdmap e876: 8 total, 8 up, 8 in 2024-03-29T05:08:43.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:42 smithi181 ceph-mon[101970]: failed to encode map e876 with expected crc 2024-03-29T05:08:43.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:42 smithi181 ceph-mon[101970]: failed to encode map e876 with expected crc 2024-03-29T05:08:43.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:42 smithi181 ceph-mon[101970]: failed to encode map e876 with expected crc 2024-03-29T05:08:44.060 INFO:tasks.workunit.client.0.smithi181.stdout: i=16 exists [404568~171768,607349~11231025,13801673~10588900,27077192~11354189,39741677~12253486,53365114~662488,56829493~11697280,72033937~3808082,78045251~13489622,93785211~10801696,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336800196~81978729] wrote [1721581~4820469,53365114~662488,81195022~6857628,108145176~934353,114910182~5114642,323647203~3993690,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870] 2024-03-29T05:08:44.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:43 smithi187 ceph-mon[93536]: pgmap v1042: 105 pgs: 105 active+clean; 654 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 189 KiB/s rd, 57 MiB/s wr, 308 op/s 2024-03-29T05:08:44.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:43 smithi187 ceph-mon[93536]: failed to encode map e877 with expected crc 2024-03-29T05:08:44.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:43 smithi187 ceph-mon[93536]: osdmap e877: 8 total, 8 up, 8 in 2024-03-29T05:08:44.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:43 smithi187 ceph-mon[93536]: failed to encode map e877 with expected crc 2024-03-29T05:08:44.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:43 smithi187 ceph-mon[93536]: failed to encode map e877 with expected crc 2024-03-29T05:08:44.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:43 smithi181 ceph-mon[101970]: pgmap v1042: 105 pgs: 105 active+clean; 654 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 189 KiB/s rd, 57 MiB/s wr, 308 op/s 2024-03-29T05:08:44.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:43 smithi181 ceph-mon[101970]: failed to encode map e877 with expected crc 2024-03-29T05:08:44.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:43 smithi181 ceph-mon[101970]: osdmap e877: 8 total, 8 up, 8 in 2024-03-29T05:08:44.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:43 smithi181 ceph-mon[101970]: failed to encode map e877 with expected crc 2024-03-29T05:08:44.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:43 smithi181 ceph-mon[101970]: failed to encode map e877 with expected crc 2024-03-29T05:08:44.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:43 smithi181 ceph-mon[106466]: pgmap v1042: 105 pgs: 105 active+clean; 654 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 189 KiB/s rd, 57 MiB/s wr, 308 op/s 2024-03-29T05:08:44.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:43 smithi181 ceph-mon[106466]: failed to encode map e877 with expected crc 2024-03-29T05:08:44.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:43 smithi181 ceph-mon[106466]: osdmap e877: 8 total, 8 up, 8 in 2024-03-29T05:08:44.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:43 smithi181 ceph-mon[106466]: failed to encode map e877 with expected crc 2024-03-29T05:08:44.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:43 smithi181 ceph-mon[106466]: failed to encode map e877 with expected crc 2024-03-29T05:08:45.091 INFO:tasks.workunit.client.0.smithi181.stdout: i=17 exists [404568~171768,607349~11231025,13801673~10588900,27077192~11354189,39741677~12253486,53365114~662488,56829493~11697280,72033937~3808082,78045251~26541656,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~16887296,292525779~1466056,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336122526~82656399] wrote [88080139~7913273,176445827~7860687,208104136~550405,211369000~1548772,256353754~7451407,269267167~7063560,318129639~337373,336122526~7073563,347366098~139655] 2024-03-29T05:08:45.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:44 smithi187 ceph-mon[93536]: failed to encode map e878 with expected crc 2024-03-29T05:08:45.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:44 smithi187 ceph-mon[93536]: osdmap e878: 8 total, 8 up, 8 in 2024-03-29T05:08:45.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:44 smithi187 ceph-mon[93536]: failed to encode map e878 with expected crc 2024-03-29T05:08:45.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:44 smithi187 ceph-mon[93536]: failed to encode map e878 with expected crc 2024-03-29T05:08:45.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:44 smithi181 ceph-mon[106466]: failed to encode map e878 with expected crc 2024-03-29T05:08:45.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:44 smithi181 ceph-mon[106466]: osdmap e878: 8 total, 8 up, 8 in 2024-03-29T05:08:45.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:44 smithi181 ceph-mon[106466]: failed to encode map e878 with expected crc 2024-03-29T05:08:45.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:44 smithi181 ceph-mon[106466]: failed to encode map e878 with expected crc 2024-03-29T05:08:45.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:44 smithi181 ceph-mon[101970]: failed to encode map e878 with expected crc 2024-03-29T05:08:45.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:44 smithi181 ceph-mon[101970]: osdmap e878: 8 total, 8 up, 8 in 2024-03-29T05:08:45.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:44 smithi181 ceph-mon[101970]: failed to encode map e878 with expected crc 2024-03-29T05:08:45.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:44 smithi181 ceph-mon[101970]: failed to encode map e878 with expected crc 2024-03-29T05:08:46.062 INFO:tasks.workunit.client.0.smithi181.stdout: i=18 exists [404568~171768,607349~11231025,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78045251~26541656,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~16887296,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~5127042,329343811~2701118,333824662~117846,336122526~82656399] wrote [46870117~7046614,59960135~3710142,78594274~2135316,85708102~6172979,120464135~4299057,195709844~6886032,298106400~3193945,304147414~1639746,333824662~117846,405770812~985542] 2024-03-29T05:08:46.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:45 smithi187 ceph-mon[93536]: pgmap v1045: 105 pgs: 105 active+clean; 696 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 227 KiB/s rd, 53 MiB/s wr, 360 op/s 2024-03-29T05:08:46.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:45 smithi187 ceph-mon[93536]: failed to encode map e879 with expected crc 2024-03-29T05:08:46.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:45 smithi187 ceph-mon[93536]: osdmap e879: 8 total, 8 up, 8 in 2024-03-29T05:08:46.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:45 smithi187 ceph-mon[93536]: failed to encode map e879 with expected crc 2024-03-29T05:08:46.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:45 smithi187 ceph-mon[93536]: failed to encode map e879 with expected crc 2024-03-29T05:08:46.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:45 smithi187 ceph-mon[93536]: failed to encode map e879 with expected crc 2024-03-29T05:08:46.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:45 smithi181 ceph-mon[101970]: pgmap v1045: 105 pgs: 105 active+clean; 696 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 227 KiB/s rd, 53 MiB/s wr, 360 op/s 2024-03-29T05:08:46.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:45 smithi181 ceph-mon[101970]: failed to encode map e879 with expected crc 2024-03-29T05:08:46.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:45 smithi181 ceph-mon[101970]: osdmap e879: 8 total, 8 up, 8 in 2024-03-29T05:08:46.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:45 smithi181 ceph-mon[101970]: failed to encode map e879 with expected crc 2024-03-29T05:08:46.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:45 smithi181 ceph-mon[101970]: failed to encode map e879 with expected crc 2024-03-29T05:08:46.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:45 smithi181 ceph-mon[101970]: failed to encode map e879 with expected crc 2024-03-29T05:08:46.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:45 smithi181 ceph-mon[106466]: pgmap v1045: 105 pgs: 105 active+clean; 696 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 227 KiB/s rd, 53 MiB/s wr, 360 op/s 2024-03-29T05:08:46.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:45 smithi181 ceph-mon[106466]: failed to encode map e879 with expected crc 2024-03-29T05:08:46.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:45 smithi181 ceph-mon[106466]: osdmap e879: 8 total, 8 up, 8 in 2024-03-29T05:08:46.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:45 smithi181 ceph-mon[106466]: failed to encode map e879 with expected crc 2024-03-29T05:08:46.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:45 smithi181 ceph-mon[106466]: failed to encode map e879 with expected crc 2024-03-29T05:08:46.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:45 smithi181 ceph-mon[106466]: failed to encode map e879 with expected crc 2024-03-29T05:08:47.071 INFO:tasks.workunit.client.0.smithi181.stdout: i=19 exists [404568~171768,607349~11231025,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78045251~26541656,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] wrote [80721595~595191,251771759~3101503,286060646~3118392,309777748~1877260,324576755~5327126,333726986~3732200,367522990~5006359,392541912~715499,398460268~2471818] 2024-03-29T05:08:47.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:46 smithi187 ceph-mon[93536]: failed to encode map e880 with expected crc 2024-03-29T05:08:47.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:46 smithi187 ceph-mon[93536]: osdmap e880: 8 total, 8 up, 8 in 2024-03-29T05:08:47.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:46 smithi187 ceph-mon[93536]: failed to encode map e880 with expected crc 2024-03-29T05:08:47.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:46 smithi187 ceph-mon[93536]: failed to encode map e880 with expected crc 2024-03-29T05:08:47.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:46 smithi187 ceph-mon[93536]: failed to encode map e880 with expected crc 2024-03-29T05:08:47.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:46 smithi181 ceph-mon[106466]: failed to encode map e880 with expected crc 2024-03-29T05:08:47.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:46 smithi181 ceph-mon[106466]: osdmap e880: 8 total, 8 up, 8 in 2024-03-29T05:08:47.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:46 smithi181 ceph-mon[106466]: failed to encode map e880 with expected crc 2024-03-29T05:08:47.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:46 smithi181 ceph-mon[106466]: failed to encode map e880 with expected crc 2024-03-29T05:08:47.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:46 smithi181 ceph-mon[106466]: failed to encode map e880 with expected crc 2024-03-29T05:08:47.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:46 smithi181 ceph-mon[101970]: failed to encode map e880 with expected crc 2024-03-29T05:08:47.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:46 smithi181 ceph-mon[101970]: osdmap e880: 8 total, 8 up, 8 in 2024-03-29T05:08:47.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:46 smithi181 ceph-mon[101970]: failed to encode map e880 with expected crc 2024-03-29T05:08:47.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:46 smithi181 ceph-mon[101970]: failed to encode map e880 with expected crc 2024-03-29T05:08:47.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:46 smithi181 ceph-mon[101970]: failed to encode map e880 with expected crc 2024-03-29T05:08:48.027 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 1 diff [30979066~7452315,79456923~3045340,156036112~645685,180137322~2342320,197250192~5963915,242552251~2998060,276551166~4724153,350452960~5753165,370473318~7189972] 2024-03-29T05:08:48.027 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [30979066~7452315,79456923~3045340,156036112~645685,180137322~2342320,197250192~5963915,242552251~2998060,276551166~4724153,350452960~5753165,370473318~7189972] 2024-03-29T05:08:48.045 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [29360128~9071253,79456923~3045340,155189248~1492549,180137322~2342320,197132288~6081819,239075328~6474983,272629760~8645559,348127232~8078893,369098752~8564538] 2024-03-29T05:08:48.045 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 2 diff [30979066~7452315,60821403~7408954,72033937~675494,79456923~3045340,112225849~4586789,156036112~645685,180137322~2342320,197250192~5963915,230584791~6738910,242552251~2998060,248873025~5038402,254279421~1816680,276551166~4724153,292525779~1466056,322822755~1665676,350452960~5753165,370473318~7189972,382386613~4399674] 2024-03-29T05:08:48.045 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [30979066~7452315,60821403~7408954,72033937~675494,79456923~3045340,112225849~4586789,156036112~645685,180137322~2342320,197250192~5963915,230584791~6738910,242552251~2998060,248873025~5038402,254279421~1816680,276551166~4724153,292525779~1466056,322822755~1665676,350452960~5753165,370473318~7189972,382386613~4399674] 2024-03-29T05:08:48.052 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [29360128~9071253,58720256~9510101,71303168~1406263,79456923~3045340,109051904~7760734,155189248~1492549,180137322~2342320,197132288~6081819,226492416~10831285,239075328~6474983,247463936~8632165,272629760~8645559,289406976~4584859,318767104~5721327,348127232~8078893,369098752~8564538,381681664~5104623] 2024-03-29T05:08:48.052 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 3 diff [30979066~7452315,41252356~6856493,56829493~11400864,72033937~675494,79456923~3045340,112225849~4586789,156036112~645685,180137322~2342320,197250192~5963915,205369725~2447469,230584791~6738910,242552251~2998060,248873025~5038402,254279421~1816680,276551166~4724153,284493911~742072,292525779~1466056,322822755~1665676,329448555~32850,340503058~21662517,370473318~7189972,382386613~4399674,406124584~7978002] 2024-03-29T05:08:48.052 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [30979066~7452315,41252356~6856493,56829493~11400864,72033937~675494,79456923~3045340,112225849~4586789,156036112~645685,180137322~2342320,197250192~5963915,205369725~2447469,230584791~6738910,242552251~2998060,248873025~5038402,254279421~1816680,276551166~4724153,284493911~742072,292525779~1466056,322822755~1665676,329448555~32850,340503058~21662517,370473318~7189972,382386613~4399674,406124584~7978002] 2024-03-29T05:08:48.058 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [29360128~18748721,54525952~13704405,71303168~1406263,79456923~3045340,109051904~7760734,155189248~1492549,180137322~2342320,197132288~10684906,226492416~10831285,239075328~6474983,247463936~8632165,272629760~12606223,289406976~4584859,318767104~5721327,327155712~2325693,339738624~22426951,369098752~8564538,381681664~5104623,405535603~8566983] 2024-03-29T05:08:48.058 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 4 diff [30979066~7452315,41252356~6856493,56829493~11400864,72033937~675494,79341797~5078800,86382691~5152182,112225849~4586789,156036112~645685,167413531~1796768,173109571~9370071,197250192~5963915,205369725~2447469,230584791~9848925,242552251~2998060,248873025~5038402,254279421~1816680,276551166~7515111,284493911~742072,292525779~1466056,322822755~1665676,329448555~32850,340503058~21662517,369627346~8035944,382386613~4399674,406124584~7978002] 2024-03-29T05:08:48.058 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [30979066~7452315,41252356~6856493,56829493~11400864,72033937~675494,79341797~5078800,86382691~5152182,112225849~4586789,156036112~645685,167413531~1796768,173109571~9370071,197250192~5963915,205369725~2447469,230584791~9848925,242552251~2998060,248873025~5038402,254279421~1816680,276551166~7515111,284493911~742072,292525779~1466056,322822755~1665676,329448555~32850,340503058~21662517,369627346~8035944,382386613~4399674,406124584~7978002] 2024-03-29T05:08:48.064 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [29360128~18748721,54525952~13704405,71303168~1406263,79341797~5078800,84790916~6743957,109051904~7760734,155189248~1492549,164536940~4673359,171966464~10513178,197132288~10684906,226492416~19057895,247463936~8632165,272629760~12606223,289406976~4584859,318767104~5721327,327155712~2325693,339738624~22426951,369098752~8564538,381681664~5104623,405535603~8566983] 2024-03-29T05:08:48.065 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 5 diff [30979066~7452315,41252356~8883318,56829493~11400864,72033937~675494,79341797~5078800,86382691~5152182,111013211~305386,112225849~4586789,156036112~645685,161975827~3924913,167413531~1796768,173109571~9370071,197250192~5963915,205369725~2447469,214230671~974716,230584791~9848925,242552251~2998060,248873025~5038402,254279421~1816680,276551166~9603297,292525779~1466056,322822755~1665676,329448555~32850,340503058~26858463,369627346~8035944,378273435~8512852,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.065 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [30979066~7452315,41252356~8883318,56829493~11400864,72033937~675494,79341797~5078800,86382691~5152182,111013211~305386,112225849~4586789,156036112~645685,161975827~3924913,167413531~1796768,173109571~9370071,197250192~5963915,205369725~2447469,214230671~974716,230584791~9848925,242552251~2998060,248873025~5038402,254279421~1816680,276551166~9603297,292525779~1466056,322822755~1665676,329448555~32850,340503058~26858463,369627346~8035944,378273435~8512852,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.071 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [29360128~20775546,54525952~13704405,71303168~1406263,79341797~5078800,84790916~6743957,109051904~7760734,155189248~1492549,161975827~7234472,171966464~10513178,197132288~10684906,213909504~1295883,226492416~19057895,247463936~8632165,272629760~13524703,289406976~4584859,318767104~5721327,327155712~2325693,339738624~27622897,369098752~17687535,403089332~1347239,405535603~8566983] 2024-03-29T05:08:48.071 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 6 diff [404568~171768,2505262~8011646,30979066~7452315,41252356~8883318,56829493~11400864,72033937~675494,79341797~5078800,86382691~5152182,98583688~6003219,111013211~305386,112225849~4586789,139874896~9256679,156036112~645685,161975827~3924913,167413531~1796768,173109571~9370071,197250192~5963915,205369725~2447469,214230671~974716,230584791~9848925,242552251~2998060,247509107~8586994,276551166~9603297,292525779~1466056,317479958~508825,322822755~1665676,329448555~32850,340503058~26858463,369627346~8035944,378273435~8512852,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.071 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,30979066~7452315,41252356~8883318,56829493~11400864,72033937~675494,79341797~5078800,86382691~5152182,98583688~6003219,111013211~305386,112225849~4586789,139874896~9256679,156036112~645685,161975827~3924913,167413531~1796768,173109571~9370071,197250192~5963915,205369725~2447469,214230671~974716,230584791~9848925,242552251~2998060,247509107~8586994,276551166~9603297,292525779~1466056,317479958~508825,322822755~1665676,329448555~32850,340503058~26858463,369627346~8035944,378273435~8512852,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.077 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,29360128~20775546,54525952~13704405,71303168~1406263,79341797~5078800,84790916~6743957,96468992~8117915,109051904~7760734,138412032~10719543,155189248~1492549,161975827~7234472,171966464~10513178,197132288~10684906,213909504~1295883,226492416~19057895,247463936~8632165,272629760~13524703,289406976~4584859,314572800~3415983,318767104~5721327,327155712~2325693,339738624~27622897,369098752~17687535,403089332~1347239,405535603~8566983] 2024-03-29T05:08:48.077 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 7 diff [404568~171768,2505262~8011646,30979066~7452315,41252356~8883318,56829493~11400864,72033937~675494,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,111013211~305386,112225849~4586789,139874896~9256679,156036112~645685,161975827~3924913,167413531~1796768,173109571~9370071,186720600~9465340,197250192~11609484,214230671~974716,230584791~9848925,242552251~2998060,247509107~10202908,276551166~9603297,292525779~1466056,312283221~8455627,322822755~1665676,329448555~32850,340503058~26858463,369627346~8035944,378273435~8512852,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.077 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,30979066~7452315,41252356~8883318,56829493~11400864,72033937~675494,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,111013211~305386,112225849~4586789,139874896~9256679,156036112~645685,161975827~3924913,167413531~1796768,173109571~9370071,186720600~9465340,197250192~11609484,214230671~974716,230584791~9848925,242552251~2998060,247509107~10202908,276551166~9603297,292525779~1466056,312283221~8455627,322822755~1665676,329448555~32850,340503058~26858463,369627346~8035944,378273435~8512852,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.083 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,29360128~20775546,54525952~13704405,71303168~4538851,79341797~5078800,84790916~6743957,96468992~8117915,109051904~7760734,138412032~10719543,155189248~1492549,161975827~7234472,171966464~10513178,186720600~9465340,197132288~11727388,213909504~1295883,226492416~19057895,247463936~10248079,272629760~13524703,289406976~4584859,310378496~14109935,327155712~2325693,339738624~27622897,369098752~17687535,403089332~1347239,405535603~8566983] 2024-03-29T05:08:48.083 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 8 diff [404568~171768,2505262~8011646,13801673~6528238,30979066~7452315,41252356~10121051,56829493~11452450,72033937~675494,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,112225849~4586789,124550033~2194459,139874896~9256679,156036112~645685,161975827~3924913,167413531~1796768,173109571~23076369,197250192~11609484,214230671~974716,230584791~9848925,242552251~2998060,246836530~10875485,276551166~9603297,292525779~1466056,312283221~8455627,322822755~1665676,329448555~32850,340503058~26858463,369627346~8035944,378273435~8512852,401507631~17271294] 2024-03-29T05:08:48.083 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,30979066~7452315,41252356~10121051,56829493~11452450,72033937~675494,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,112225849~4586789,124550033~2194459,139874896~9256679,156036112~645685,161975827~3924913,167413531~1796768,173109571~23076369,197250192~11609484,214230671~974716,230584791~9848925,242552251~2998060,246836530~10875485,276551166~9603297,292525779~1466056,312283221~8455627,322822755~1665676,329448555~32850,340503058~26858463,369627346~8035944,378273435~8512852,401507631~17271294] 2024-03-29T05:08:48.089 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,29360128~22013279,54525952~13755991,71303168~4538851,79341797~5078800,84790916~6743957,96468992~8117915,104857600~3451536,109051904~7760734,121634816~5109676,138412032~10719543,155189248~1492549,161975827~7234472,171966464~24219476,197132288~11727388,213909504~1295883,226492416~31219599,272629760~13524703,289406976~4584859,310378496~14109935,327155712~2325693,339738624~27622897,369098752~17687535,398458880~20320045] 2024-03-29T05:08:48.089 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 9 diff [404568~171768,2505262~8011646,13801673~6528238,30979066~7452315,39741677~11631730,56829493~11452450,72033937~675494,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,112225849~4586789,124550033~2194459,139874896~9256679,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,214230671~974716,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,276551166~9603297,292525779~1466056,312283221~8455627,322822755~1665676,329448555~32850,340503058~26858463,369627346~8035944,378273435~10682903,397672230~21106695] 2024-03-29T05:08:48.090 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,30979066~7452315,39741677~11631730,56829493~11452450,72033937~675494,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,112225849~4586789,124550033~2194459,139874896~9256679,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,214230671~974716,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,276551166~9603297,292525779~1466056,312283221~8455627,322822755~1665676,329448555~32850,340503058~26858463,369627346~8035944,378273435~10682903,397672230~21106695] 2024-03-29T05:08:48.095 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,29360128~22013279,54525952~13755991,71303168~4538851,79341797~5078800,84790916~6743957,96468992~8117915,104857600~3451536,109051904~7760734,121634816~5109676,138412032~10719543,155189248~1492549,161975827~7234472,171966464~39706493,213909504~1295883,226492416~31219599,260046848~5747696,272629760~13524703,289406976~4584859,310378496~14109935,327155712~2325693,339738624~27622897,369098752~19857586,394264576~24514349] 2024-03-29T05:08:48.095 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 10 diff [404568~171768,2505262~8011646,13801673~6528238,30979066~7452315,39741677~11631730,56829493~11452450,72033937~3808082,79341797~5110550,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~9256679,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,214230671~974716,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,276551166~9603297,292525779~1466056,295753178~3458866,312283221~8455627,322680407~1808024,329448555~32850,340503058~26858463,369311906~8351384,378273435~10682903,397672230~21106695] 2024-03-29T05:08:48.095 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,30979066~7452315,39741677~11631730,56829493~11452450,72033937~3808082,79341797~5110550,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~9256679,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,214230671~974716,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,276551166~9603297,292525779~1466056,295753178~3458866,312283221~8455627,322680407~1808024,329448555~32850,340503058~26858463,369311906~8351384,378273435~10682903,397672230~21106695] 2024-03-29T05:08:48.101 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,29360128~22013279,54525952~13755991,71303168~4538851,79341797~5110550,84790916~6743957,96468992~8117915,104857600~3451536,109051904~9479431,121634816~5109676,138412032~10719543,155189248~1492549,161975827~7234472,171966464~39706493,213909504~1295883,226492416~31219599,260046848~5747696,272629760~13524703,289406976~9805068,310378496~14109935,327155712~2325693,339738624~27622897,369098752~19857586,394264576~24514349] 2024-03-29T05:08:48.101 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 11 diff [404568~171768,2505262~8011646,13801673~6528238,30979066~7452315,39741677~11631730,56829493~11452450,72033937~3808082,79341797~5110550,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,276551166~9603297,292525779~1466056,295753178~3458866,312283221~8455627,322513851~1989805,329343811~2701118,340147706~27213815,369311906~8351384,378273435~16507122,397672230~21106695] 2024-03-29T05:08:48.101 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,30979066~7452315,39741677~11631730,56829493~11452450,72033937~3808082,79341797~5110550,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,276551166~9603297,292525779~1466056,295753178~3458866,312283221~8455627,322513851~1989805,329343811~2701118,340147706~27213815,369311906~8351384,378273435~16507122,397672230~21106695] 2024-03-29T05:08:48.107 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,29360128~22013279,54525952~13755991,71303168~4538851,79341797~5110550,84790916~6743957,96468992~8117915,104857600~3451536,109051904~9479431,121634816~5109676,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~31219599,260046848~5747696,272629760~13524703,289406976~9805068,310378496~14125160,327155712~4889217,339738624~27622897,369098752~49680173] 2024-03-29T05:08:48.107 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 12 diff [404568~171768,2505262~8011646,13801673~6528238,28709632~1902975,30979066~7452315,39741677~11631730,56829493~11452450,72033937~3808082,78839889~5612458,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,340147706~27213815,369311906~8351384,378273435~16507122,397672230~21106695] 2024-03-29T05:08:48.107 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,28709632~1902975,30979066~7452315,39741677~11631730,56829493~11452450,72033937~3808082,78839889~5612458,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,340147706~27213815,369311906~8351384,378273435~16507122,397672230~21106695] 2024-03-29T05:08:48.113 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,25165824~26207583,54525952~13755991,71303168~4538851,78839889~5612458,84790916~6743957,96468992~8117915,104857600~3451536,109051904~9479431,121634816~5109676,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~31219599,260046848~5747696,272629760~13524703,289406976~9805068,306184192~18319464,327155712~4889217,339738624~27622897,369098752~49680173] 2024-03-29T05:08:48.113 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 13 diff [404568~171768,2505262~8011646,13801673~10588900,28709632~1902975,30979066~7452315,39741677~11631730,56829493~11697280,72033937~3808082,78839889~5612458,86382691~5152182,98583688~6003219,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~40863094,378273435~40505490] 2024-03-29T05:08:48.113 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~10588900,28709632~1902975,30979066~7452315,39741677~11631730,56829493~11697280,72033937~3808082,78839889~5612458,86382691~5152182,98583688~6003219,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~40863094,378273435~40505490] 2024-03-29T05:08:48.119 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~26207583,54525952~14000821,71303168~4538851,78839889~5612458,84790916~6743957,96468992~8117915,104857600~3451536,109051904~9479431,121634816~13523600,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~31219599,260046848~5747696,272629760~13524703,289406976~9805068,306184192~18319464,327155712~4889217,335544320~83234605] 2024-03-29T05:08:48.119 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 14 diff [404568~171768,2505262~8011646,13801673~10588900,27077192~11354189,39741677~11631730,56829493~11697280,72033937~3808082,78839889~5612458,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.119 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~10588900,27077192~11354189,39741677~11631730,56829493~11697280,72033937~3808082,78839889~5612458,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.125 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~26207583,54525952~14000821,71303168~4538851,78839889~5612458,84790916~6743957,92274688~12312219,104857600~3451536,109051904~9479431,121634816~13523600,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~31219599,260046848~5747696,272629760~13524703,289406976~9805068,306184192~18319464,327155712~4889217,335544320~83234605] 2024-03-29T05:08:48.125 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 15 diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~12253486,56829493~11697280,72033937~3808082,78839889~5612458,84964601~734981,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.125 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~12253486,56829493~11697280,72033937~3808082,78839889~5612458,84964601~734981,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.131 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~26829339,54525952~14000821,71303168~4538851,78839889~5612458,84790916~6743957,92274688~12312219,104857600~3451536,109051904~9479431,121634816~13523600,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~31219599,260046848~5747696,272629760~13524703,289406976~9805068,306184192~18319464,327155712~4889217,335544320~83234605] 2024-03-29T05:08:48.131 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 16 diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~12253486,53365114~662488,56829493~11697280,72033937~3808082,78839889~12694984,93785211~10801696,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.131 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~12253486,53365114~662488,56829493~11697280,72033937~3808082,78839889~12694984,93785211~10801696,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,230584791~9848925,242552251~2998060,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.136 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78839889~12694984,92274688~12312219,104857600~15167224,121634816~13523600,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~31219599,260046848~5747696,272629760~13524703,289406976~9805068,306184192~25860737,335544320~83234605] 2024-03-29T05:08:48.136 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 17 diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~12253486,53365114~662488,56829493~11697280,72033937~3808082,78839889~25747018,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~16887296,292525779~1466056,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336122526~82656399] 2024-03-29T05:08:48.137 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~12253486,53365114~662488,56829493~11697280,72033937~3808082,78839889~25747018,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~16887296,292525779~1466056,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336122526~82656399] 2024-03-29T05:08:48.142 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78839889~25747018,104857600~15167224,121634816~13523600,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~17719007,289406976~9805068,306184192~25860737,335544320~83234605] 2024-03-29T05:08:48.142 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 18 diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~16887296,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~5127042,329343811~2701118,333824662~117846,336122526~82656399] 2024-03-29T05:08:48.142 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~16887296,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~5127042,329343811~2701118,333824662~117846,336122526~82656399] 2024-03-29T05:08:48.150 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~17719007,289406976~11893369,301989888~3797272,306184192~27758316,335544320~83234605] 2024-03-29T05:08:48.150 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 19 diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:48.150 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:48.154 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:48.154 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 2 diff [60821403~7408954,72033937~675494,112225849~4586789,230584791~6738910,248873025~5038402,254279421~1816680,292525779~1466056,322822755~1665676,382386613~4399674] 2024-03-29T05:08:48.154 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [60821403~7408954,72033937~675494,112225849~4586789,230584791~6738910,248873025~5038402,254279421~1816680,292525779~1466056,322822755~1665676,382386613~4399674] 2024-03-29T05:08:48.160 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [58720256~9510101,71303168~1406263,109051904~7760734,226492416~10831285,247463936~8632165,289406976~4584859,318767104~5721327,381681664~5104623] 2024-03-29T05:08:48.160 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 3 diff [41252356~6856493,56829493~11400864,72033937~675494,112225849~4586789,205369725~2447469,230584791~6738910,248873025~5038402,254279421~1816680,284493911~742072,292525779~1466056,322822755~1665676,329448555~32850,340503058~12114163,355236756~6928819,371174992~1402638,382386613~4399674,406124584~7978002] 2024-03-29T05:08:48.160 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [41252356~6856493,56829493~11400864,72033937~675494,112225849~4586789,205369725~2447469,230584791~6738910,248873025~5038402,254279421~1816680,284493911~742072,292525779~1466056,322822755~1665676,329448555~32850,340503058~12114163,355236756~6928819,371174992~1402638,382386613~4399674,406124584~7978002] 2024-03-29T05:08:48.165 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [38431381~9677468,54525952~13704405,71303168~1406263,109051904~7760734,203214107~4603087,226492416~10831285,247463936~8632165,281275319~3960664,289406976~4584859,318767104~5721327,327155712~2325693,339738624~12878597,355236756~6928819,371174992~1402638,381681664~5104623,405535603~8566983] 2024-03-29T05:08:48.165 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 4 diff [31236423~4602591,41252356~6856493,56829493~11400864,72033937~675494,79341797~5078800,86382691~5152182,112225849~4586789,167413531~1796768,173109571~9193179,205369725~2447469,230584791~9848925,248873025~5038402,254279421~1816680,276609835~7456442,284493911~742072,292525779~1466056,322822755~1665676,329448555~32850,340503058~12114163,355236756~6928819,369627346~5540348,382386613~4399674,406124584~7978002] 2024-03-29T05:08:48.166 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [31236423~4602591,41252356~6856493,56829493~11400864,72033937~675494,79341797~5078800,86382691~5152182,112225849~4586789,167413531~1796768,173109571~9193179,205369725~2447469,230584791~9848925,248873025~5038402,254279421~1816680,276609835~7456442,284493911~742072,292525779~1466056,322822755~1665676,329448555~32850,340503058~12114163,355236756~6928819,369627346~5540348,382386613~4399674,406124584~7978002] 2024-03-29T05:08:48.171 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [31236423~4602591,38431381~9677468,54525952~13704405,71303168~1406263,79341797~5078800,84790916~6743957,109051904~7760734,164536940~4673359,171966464~10336286,203214107~4603087,226492416~13941300,247463936~8632165,276609835~8626148,289406976~4584859,318767104~5721327,327155712~2325693,339738624~12878597,355236756~6928819,369627346~5540348,381681664~5104623,405535603~8566983] 2024-03-29T05:08:48.171 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 5 diff [31236423~4602591,41252356~8883318,56829493~11400864,72033937~675494,79341797~5078800,86382691~5152182,111013211~305386,112225849~4586789,161975827~3924913,167413531~1796768,173109571~9193179,205369725~2447469,214230671~974716,230584791~9848925,248873025~5038402,254279421~1816680,276609835~9544628,292525779~1466056,322822755~1665676,329448555~32850,340503058~12114163,355236756~12124765,369627346~5540348,378273435~8512852,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.171 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [31236423~4602591,41252356~8883318,56829493~11400864,72033937~675494,79341797~5078800,86382691~5152182,111013211~305386,112225849~4586789,161975827~3924913,167413531~1796768,173109571~9193179,205369725~2447469,214230671~974716,230584791~9848925,248873025~5038402,254279421~1816680,276609835~9544628,292525779~1466056,322822755~1665676,329448555~32850,340503058~12114163,355236756~12124765,369627346~5540348,378273435~8512852,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.177 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [31236423~4602591,38431381~11704293,54525952~13704405,71303168~1406263,79341797~5078800,84790916~6743957,109051904~7760734,161975827~7234472,171966464~10336286,203214107~4603087,213909504~1295883,226492416~13941300,247463936~8632165,276609835~9544628,289406976~4584859,318767104~5721327,327155712~2325693,339738624~12878597,355236756~12124765,369627346~5540348,377663290~9122997,403089332~1347239,405535603~8566983] 2024-03-29T05:08:48.177 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 6 diff [404568~171768,2505262~8011646,31236423~4602591,41252356~8883318,56829493~11400864,72033937~675494,79341797~5078800,86382691~5152182,98583688~6003219,111013211~305386,112225849~4586789,139874896~9256679,161975827~3924913,167413531~1796768,173109571~9193179,205369725~2447469,214230671~974716,230584791~9848925,247509107~8586994,276609835~9544628,292525779~1466056,317479958~508825,322822755~1665676,329448555~32850,340503058~12373579,353208019~14153502,369627346~5540348,378273435~8512852,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.177 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,31236423~4602591,41252356~8883318,56829493~11400864,72033937~675494,79341797~5078800,86382691~5152182,98583688~6003219,111013211~305386,112225849~4586789,139874896~9256679,161975827~3924913,167413531~1796768,173109571~9193179,205369725~2447469,214230671~974716,230584791~9848925,247509107~8586994,276609835~9544628,292525779~1466056,317479958~508825,322822755~1665676,329448555~32850,340503058~12373579,353208019~14153502,369627346~5540348,378273435~8512852,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.183 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,31236423~4602591,38431381~11704293,54525952~13704405,71303168~1406263,79341797~5078800,84790916~6743957,96468992~8117915,109051904~7760734,138412032~10719543,161975827~7234472,171966464~10336286,203214107~4603087,213909504~1295883,226492416~13941300,247463936~8632165,276609835~9544628,289406976~4584859,314572800~3415983,318767104~5721327,327155712~2325693,339738624~13138013,353208019~14153502,369627346~5540348,377663290~9122997,403089332~1347239,405535603~8566983] 2024-03-29T05:08:48.183 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 7 diff [404568~171768,2505262~8011646,31236423~4602591,41252356~8883318,56829493~11400864,72033937~675494,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,111013211~305386,112225849~4586789,139874896~9256679,161975827~3924913,167413531~1796768,173109571~9193179,186720600~9465340,200757268~8102408,214230671~974716,230584791~9848925,247509107~10202908,276609835~9544628,292525779~1466056,312283221~8455627,322822755~1665676,329448555~32850,340503058~12373579,353208019~14153502,369627346~5540348,378273435~8512852,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.183 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,31236423~4602591,41252356~8883318,56829493~11400864,72033937~675494,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,111013211~305386,112225849~4586789,139874896~9256679,161975827~3924913,167413531~1796768,173109571~9193179,186720600~9465340,200757268~8102408,214230671~974716,230584791~9848925,247509107~10202908,276609835~9544628,292525779~1466056,312283221~8455627,322822755~1665676,329448555~32850,340503058~12373579,353208019~14153502,369627346~5540348,378273435~8512852,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.188 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,31236423~4602591,38431381~11704293,54525952~13704405,71303168~4538851,79341797~5078800,84790916~6743957,96468992~8117915,109051904~7760734,138412032~10719543,161975827~7234472,171966464~10336286,186720600~9465340,200757268~8102408,213909504~1295883,226492416~13941300,247463936~10248079,276609835~9544628,289406976~4584859,310378496~14109935,327155712~2325693,339738624~13138013,353208019~14153502,369627346~5540348,377663290~9122997,403089332~1347239,405535603~8566983] 2024-03-29T05:08:48.188 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 8 diff [404568~171768,2505262~8011646,13801673~6528238,31236423~4602591,41252356~10121051,56829493~11452450,72033937~675494,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,112225849~4586789,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~23076369,200757268~8102408,214230671~974716,230584791~9848925,246836530~10875485,276609835~9544628,292525779~1466056,312283221~8455627,322822755~1665676,329448555~32850,340503058~12373579,353208019~14153502,369627346~5540348,378273435~8512852,401507631~17271294] 2024-03-29T05:08:48.188 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,31236423~4602591,41252356~10121051,56829493~11452450,72033937~675494,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,112225849~4586789,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~23076369,200757268~8102408,214230671~974716,230584791~9848925,246836530~10875485,276609835~9544628,292525779~1466056,312283221~8455627,322822755~1665676,329448555~32850,340503058~12373579,353208019~14153502,369627346~5540348,378273435~8512852,401507631~17271294] 2024-03-29T05:08:48.194 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,31236423~4602591,38431381~12942026,54525952~13755991,71303168~4538851,79341797~5078800,84790916~6743957,96468992~8117915,104857600~3451536,109051904~7760734,121634816~5109676,138412032~10719543,161975827~7234472,171966464~24219476,200757268~8102408,213909504~1295883,226492416~13941300,245550311~12161704,276609835~9544628,289406976~4584859,310378496~14109935,327155712~2325693,339738624~13138013,353208019~14153502,369627346~5540348,377663290~9122997,398458880~20320045] 2024-03-29T05:08:48.194 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 9 diff [404568~171768,2505262~8011646,13801673~6528238,31236423~4602591,39741677~11631730,56829493~11452450,72033937~675494,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,112225849~4586789,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,214230671~974716,230584791~9848925,246836530~10875485,263765730~2028814,276609835~9544628,292525779~1466056,312283221~8455627,322822755~1665676,329448555~32850,340503058~12373579,353208019~14153502,369627346~5540348,378273435~10682903,397672230~21106695] 2024-03-29T05:08:48.194 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,31236423~4602591,39741677~11631730,56829493~11452450,72033937~675494,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,112225849~4586789,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,214230671~974716,230584791~9848925,246836530~10875485,263765730~2028814,276609835~9544628,292525779~1466056,312283221~8455627,322822755~1665676,329448555~32850,340503058~12373579,353208019~14153502,369627346~5540348,378273435~10682903,397672230~21106695] 2024-03-29T05:08:48.200 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,31236423~4602591,38431381~12942026,54525952~13755991,71303168~4538851,79341797~5078800,84790916~6743957,96468992~8117915,104857600~3451536,109051904~7760734,121634816~5109676,138412032~10719543,161975827~7234472,171966464~26626708,200757268~10915689,213909504~1295883,226492416~13941300,245550311~12161704,260046848~5747696,276609835~9544628,289406976~4584859,310378496~14109935,327155712~2325693,339738624~13138013,353208019~14153502,369627346~5540348,377663290~11293048,394264576~24514349] 2024-03-29T05:08:48.200 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 10 diff [404568~171768,2505262~8011646,13801673~6528238,31236423~6140372,39741677~11631730,56829493~11452450,72033937~3808082,79341797~5110550,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,214230671~974716,230584791~9848925,246836530~10875485,263765730~2028814,276609835~9544628,292525779~1466056,295753178~3458866,312283221~8455627,322680407~1808024,329448555~32850,340503058~12373579,353208019~14153502,369311906~8107396,378273435~10682903,397672230~21106695] 2024-03-29T05:08:48.200 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,31236423~6140372,39741677~11631730,56829493~11452450,72033937~3808082,79341797~5110550,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,214230671~974716,230584791~9848925,246836530~10875485,263765730~2028814,276609835~9544628,292525779~1466056,295753178~3458866,312283221~8455627,322680407~1808024,329448555~32850,340503058~12373579,353208019~14153502,369311906~8107396,378273435~10682903,397672230~21106695] 2024-03-29T05:08:48.205 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,31236423~6140372,38431381~12942026,54525952~13755991,71303168~4538851,79341797~5110550,84790916~6743957,96468992~8117915,104857600~3451536,109051904~9479431,121634816~5109676,138412032~10719543,161975827~7234472,171966464~26626708,200757268~10915689,213909504~1295883,226492416~13941300,245550311~12161704,260046848~5747696,276609835~9544628,289406976~9805068,310378496~14109935,327155712~2325693,339738624~13138013,353208019~14153502,369311906~8107396,377663290~11293048,394264576~24514349] 2024-03-29T05:08:48.205 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 11 diff [404568~171768,2505262~8011646,13801673~6528238,31236423~6140372,39741677~11631730,56829493~11452450,72033937~3808082,79341797~5110550,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,213342593~7281743,230584791~9848925,246836530~10875485,263765730~2028814,276609835~9544628,292525779~1466056,295753178~3458866,312283221~8455627,322513851~1989805,329343811~2701118,340147706~12728931,353208019~14153502,369311906~8107396,378273435~16507122,397672230~21106695] 2024-03-29T05:08:48.205 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,31236423~6140372,39741677~11631730,56829493~11452450,72033937~3808082,79341797~5110550,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,213342593~7281743,230584791~9848925,246836530~10875485,263765730~2028814,276609835~9544628,292525779~1466056,295753178~3458866,312283221~8455627,322513851~1989805,329343811~2701118,340147706~12728931,353208019~14153502,369311906~8107396,378273435~16507122,397672230~21106695] 2024-03-29T05:08:48.211 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,31236423~6140372,38431381~12942026,54525952~13755991,71303168~4538851,79341797~5110550,84790916~6743957,96468992~8117915,104857600~3451536,109051904~9479431,121634816~5109676,138412032~13132147,161975827~7234472,171966464~26626708,200757268~19867068,226492416~13941300,245550311~12161704,260046848~5747696,276609835~9544628,289406976~9805068,310378496~14125160,327155712~4889217,339738624~13138013,353208019~14153502,369311906~8107396,377663290~41115635] 2024-03-29T05:08:48.211 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 12 diff [404568~171768,2505262~8011646,13801673~6528238,28709632~1902975,31236423~6140372,39741677~11631730,56829493~11452450,72033937~3808082,78839889~5612458,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,161975827~3924913,167413531~1796768,173109571~27176782,200757268~10915689,213342593~7281743,230584791~9848925,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,340147706~12728931,353208019~14153502,369311906~8107396,378273435~16507122,397672230~21106695] 2024-03-29T05:08:48.211 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,28709632~1902975,31236423~6140372,39741677~11631730,56829493~11452450,72033937~3808082,78839889~5612458,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,161975827~3924913,167413531~1796768,173109571~27176782,200757268~10915689,213342593~7281743,230584791~9848925,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,340147706~12728931,353208019~14153502,369311906~8107396,378273435~16507122,397672230~21106695] 2024-03-29T05:08:48.216 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,25165824~5446783,31236423~6140372,38431381~12942026,54525952~13755991,71303168~4538851,78839889~5612458,84790916~6743957,96468992~8117915,104857600~3451536,109051904~9479431,121634816~5109676,138412032~13132147,161975827~7234472,171966464~28319889,200757268~19867068,226492416~13941300,245550311~12161704,260046848~5747696,274173672~11980791,289406976~9805068,306184192~18319464,327155712~4889217,339738624~13138013,353208019~14153502,369311906~8107396,377663290~41115635] 2024-03-29T05:08:48.216 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 13 diff [404568~171768,2505262~8011646,13801673~10588900,28709632~1902975,31236423~6140372,39741677~11631730,56829493~11697280,72033937~3808082,78839889~5612458,86382691~5152182,98583688~6003219,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~27176782,200757268~10915689,213342593~7281743,230584791~9848925,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~16076441,353208019~24211283,378273435~40505490] 2024-03-29T05:08:48.216 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~10588900,28709632~1902975,31236423~6140372,39741677~11631730,56829493~11697280,72033937~3808082,78839889~5612458,86382691~5152182,98583688~6003219,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~27176782,200757268~10915689,213342593~7281743,230584791~9848925,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~16076441,353208019~24211283,378273435~40505490] 2024-03-29T05:08:48.222 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~5446783,31236423~6140372,38431381~12942026,54525952~14000821,71303168~4538851,78839889~5612458,84790916~6743957,96468992~8117915,104857600~3451536,109051904~9479431,121634816~13523600,138412032~13132147,161975827~7234472,171966464~28319889,200757268~19867068,226492416~13941300,245550311~12161704,260046848~5747696,274173672~11980791,289406976~9805068,306184192~18319464,327155712~4889217,335544320~17332317,353208019~24211283,377663290~41115635] 2024-03-29T05:08:48.222 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 14 diff [404568~171768,2505262~8011646,13801673~10588900,27077192~10299603,39741677~11631730,56829493~11697280,72033937~3808082,78839889~5612458,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,230584791~9848925,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.222 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~10588900,27077192~10299603,39741677~11631730,56829493~11697280,72033937~3808082,78839889~5612458,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,230584791~9848925,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.227 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12210971,38431381~12942026,54525952~14000821,71303168~4538851,78839889~5612458,84790916~6743957,92274688~12312219,104857600~3451536,109051904~9479431,121634816~13523600,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~12161704,260046848~5747696,274173672~11980791,289406976~9805068,306184192~18319464,327155712~4889217,335544320~83234605] 2024-03-29T05:08:48.228 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 15 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,56829493~11697280,72033937~3808082,78839889~5612458,84964601~734981,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,230584791~9848925,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.228 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,56829493~11697280,72033937~3808082,78839889~5612458,84964601~734981,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,230584791~9848925,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.233 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~13563782,54525952~14000821,71303168~4538851,78839889~5612458,84790916~6743957,92274688~12312219,104857600~3451536,109051904~9479431,121634816~13523600,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~12161704,260046848~5747696,274173672~11980791,289406976~9805068,306184192~18319464,327155712~4889217,335544320~83234605] 2024-03-29T05:08:48.233 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 16 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,56829493~11697280,72033937~3808082,78839889~12694984,93785211~10801696,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,230584791~9848925,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.233 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,56829493~11697280,72033937~3808082,78839889~12694984,93785211~10801696,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,230584791~9848925,246836530~10875485,263765730~2028814,274173672~11980791,292525779~1466056,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.239 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78839889~12694984,92274688~12312219,104857600~15167224,121634816~13523600,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~12161704,260046848~5747696,274173672~11980791,289406976~9805068,306184192~25860737,335544320~83234605] 2024-03-29T05:08:48.239 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 17 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,56829493~11697280,72033937~3808082,78839889~25747018,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~16887296,292525779~1466056,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336122526~82656399] 2024-03-29T05:08:48.239 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,56829493~11697280,72033937~3808082,78839889~25747018,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~16887296,292525779~1466056,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336122526~82656399] 2024-03-29T05:08:48.244 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78839889~25747018,104857600~15167224,121634816~13523600,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~17719007,289406976~9805068,306184192~25860737,335544320~83234605] 2024-03-29T05:08:48.245 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 18 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~16887296,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~5127042,329343811~2701118,333824662~117846,336122526~82656399] 2024-03-29T05:08:48.245 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~16887296,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~5127042,329343811~2701118,333824662~117846,336122526~82656399] 2024-03-29T05:08:48.250 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~17719007,289406976~11893369,301989888~3797272,306184192~27758316,335544320~83234605] 2024-03-29T05:08:48.250 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 19 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:48.250 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:48.255 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:48.255 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 3 diff [41252356~6856493,56829493~7192042,205369725~2447469,284493911~742072,329448555~32850,340503058~12114163,355236756~6928819,371174992~1402638,406124584~7978002] 2024-03-29T05:08:48.256 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [41252356~6856493,56829493~7192042,205369725~2447469,284493911~742072,329448555~32850,340503058~12114163,355236756~6928819,371174992~1402638,406124584~7978002] 2024-03-29T05:08:48.261 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [38431381~9677468,54525952~9495583,203214107~4603087,281275319~3960664,327155712~2325693,339738624~12878597,355236756~6928819,371174992~1402638,405535603~8566983] 2024-03-29T05:08:48.261 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 4 diff [31236423~4602591,41252356~6856493,56829493~7192042,66224838~1583868,79341797~5078800,86382691~5152182,167413531~1796768,173109571~9193179,205369725~2447469,232982965~7450751,276609835~7456442,284493911~742072,329448555~32850,340503058~12114163,355236756~6928819,369627346~5540348,406124584~7978002] 2024-03-29T05:08:48.261 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [31236423~4602591,41252356~6856493,56829493~7192042,66224838~1583868,79341797~5078800,86382691~5152182,167413531~1796768,173109571~9193179,205369725~2447469,232982965~7450751,276609835~7456442,284493911~742072,329448555~32850,340503058~12114163,355236756~6928819,369627346~5540348,406124584~7978002] 2024-03-29T05:08:48.266 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [31236423~4602591,38431381~9677468,54525952~9495583,66224838~1583868,79341797~5078800,84790916~6743957,164536940~4673359,171966464~10336286,203214107~4603087,232982965~7450751,276609835~8626148,327155712~2325693,339738624~12878597,355236756~6928819,369627346~5540348,405535603~8566983] 2024-03-29T05:08:48.267 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 5 diff [31236423~4602591,41252356~8883318,56829493~7192042,66224838~1583868,79341797~5078800,86382691~5152182,111013211~305386,161975827~3924913,167413531~1796768,173109571~9193179,205369725~2447469,214230671~974716,232982965~7450751,276609835~9544628,293351984~1296,329448555~32850,340503058~12114163,355236756~12124765,369627346~5540348,378273435~4517086,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.267 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [31236423~4602591,41252356~8883318,56829493~7192042,66224838~1583868,79341797~5078800,86382691~5152182,111013211~305386,161975827~3924913,167413531~1796768,173109571~9193179,205369725~2447469,214230671~974716,232982965~7450751,276609835~9544628,293351984~1296,329448555~32850,340503058~12114163,355236756~12124765,369627346~5540348,378273435~4517086,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.272 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [31236423~4602591,38431381~11704293,54525952~9495583,66224838~1583868,79341797~5078800,84790916~6743957,111013211~305386,161975827~7234472,171966464~10336286,203214107~4603087,213909504~1295883,232982965~7450751,276609835~9544628,293351984~1296,327155712~2325693,339738624~12878597,355236756~12124765,369627346~5540348,377663290~5127231,403089332~1347239,405535603~8566983] 2024-03-29T05:08:48.272 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 6 diff [404568~171768,2505262~8011646,31236423~4602591,41252356~8883318,56829493~7192042,66224838~1583868,79341797~5078800,86382691~5152182,98583688~6003219,111013211~305386,139874896~9256679,161975827~3924913,167413531~1796768,173109571~9193179,205369725~2447469,214230671~974716,232982965~7450751,247509107~7937298,276609835~9544628,293351984~1296,317479958~508825,329448555~32850,340503058~12373579,353208019~14153502,369627346~5540348,378273435~4517086,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.272 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,31236423~4602591,41252356~8883318,56829493~7192042,66224838~1583868,79341797~5078800,86382691~5152182,98583688~6003219,111013211~305386,139874896~9256679,161975827~3924913,167413531~1796768,173109571~9193179,205369725~2447469,214230671~974716,232982965~7450751,247509107~7937298,276609835~9544628,293351984~1296,317479958~508825,329448555~32850,340503058~12373579,353208019~14153502,369627346~5540348,378273435~4517086,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.278 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,31236423~4602591,38431381~11704293,54525952~9495583,66224838~1583868,79341797~5078800,84790916~6743957,96468992~8117915,111013211~305386,138412032~10719543,161975827~7234472,171966464~10336286,203214107~4603087,213909504~1295883,232982965~7450751,247509107~7937298,276609835~9544628,293351984~1296,314572800~3415983,327155712~2325693,339738624~13138013,353208019~14153502,369627346~5540348,377663290~5127231,403089332~1347239,405535603~8566983] 2024-03-29T05:08:48.278 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 7 diff [404568~171768,2505262~8011646,31236423~4602591,41252356~8883318,56829493~8603262,66224838~1583868,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,111013211~305386,139874896~9256679,161975827~3924913,167413531~1796768,173109571~9193179,186720600~9465340,200757268~8102408,214230671~974716,232982965~7450751,247509107~10202908,276609835~9544628,293351984~1296,312283221~8455627,329448555~32850,340503058~12373579,353208019~14153502,369627346~5540348,378273435~4517086,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.278 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,31236423~4602591,41252356~8883318,56829493~8603262,66224838~1583868,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,111013211~305386,139874896~9256679,161975827~3924913,167413531~1796768,173109571~9193179,186720600~9465340,200757268~8102408,214230671~974716,232982965~7450751,247509107~10202908,276609835~9544628,293351984~1296,312283221~8455627,329448555~32850,340503058~12373579,353208019~14153502,369627346~5540348,378273435~4517086,403089332~1347239,406124584~7978002] 2024-03-29T05:08:48.283 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,31236423~4602591,38431381~11704293,54525952~10906803,66224838~1583868,72709431~3132588,79341797~5078800,84790916~6743957,96468992~8117915,111013211~305386,138412032~10719543,161975827~7234472,171966464~10336286,186720600~9465340,200757268~8102408,213909504~1295883,232982965~7450751,247509107~10202908,276609835~9544628,293351984~1296,310378496~10360352,327155712~2325693,339738624~13138013,353208019~14153502,369627346~5540348,377663290~5127231,403089332~1347239,405535603~8566983] 2024-03-29T05:08:48.283 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 8 diff [404568~171768,2505262~8011646,13801673~6528238,31236423~4602591,41252356~10121051,56829493~11452450,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~23076369,200757268~8102408,214230671~974716,232982965~7450751,246836530~10875485,276609835~9544628,293351984~1296,312283221~8455627,329448555~32850,340503058~12373579,353208019~14153502,369627346~5540348,378273435~4517086,401507631~17271294] 2024-03-29T05:08:48.283 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,31236423~4602591,41252356~10121051,56829493~11452450,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~23076369,200757268~8102408,214230671~974716,232982965~7450751,246836530~10875485,276609835~9544628,293351984~1296,312283221~8455627,329448555~32850,340503058~12373579,353208019~14153502,369627346~5540348,378273435~4517086,401507631~17271294] 2024-03-29T05:08:48.289 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,31236423~4602591,38431381~12942026,54525952~13755991,72709431~3132588,79341797~5078800,84790916~6743957,96468992~8117915,104857600~3451536,111013211~305386,121634816~5109676,138412032~10719543,161975827~7234472,171966464~24219476,200757268~8102408,213909504~1295883,232982965~7450751,245550311~12161704,276609835~9544628,293351984~1296,310378496~10360352,327155712~2325693,339738624~13138013,353208019~14153502,369627346~5540348,377663290~5127231,398458880~20320045] 2024-03-29T05:08:48.289 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 9 diff [404568~171768,2505262~8011646,13801673~6528238,31236423~4602591,39741677~11631730,56829493~11452450,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,214230671~974716,232982965~7450751,246836530~10875485,263765730~2028814,276609835~9544628,293351984~1296,312283221~8455627,329448555~32850,340503058~12373579,353208019~14153502,369627346~5540348,378273435~10682903,397672230~21106695] 2024-03-29T05:08:48.289 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,31236423~4602591,39741677~11631730,56829493~11452450,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,214230671~974716,232982965~7450751,246836530~10875485,263765730~2028814,276609835~9544628,293351984~1296,312283221~8455627,329448555~32850,340503058~12373579,353208019~14153502,369627346~5540348,378273435~10682903,397672230~21106695] 2024-03-29T05:08:48.295 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,31236423~4602591,38431381~12942026,54525952~13755991,72709431~3132588,79341797~5078800,84790916~6743957,96468992~8117915,104857600~3451536,111013211~305386,121634816~5109676,138412032~10719543,161975827~7234472,171966464~26626708,200757268~10915689,213909504~1295883,232982965~7450751,245550311~12161704,260046848~5747696,276609835~9544628,293351984~1296,310378496~10360352,327155712~2325693,339738624~13138013,353208019~14153502,369627346~5540348,377663290~11293048,394264576~24514349] 2024-03-29T05:08:48.295 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 10 diff [404568~171768,2505262~8011646,13801673~6528238,31236423~6140372,39741677~11631730,56829493~11452450,72073222~3768797,79341797~5110550,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,214230671~974716,232982965~7450751,246836530~10875485,263765730~2028814,276609835~9544628,293351984~1296,295753178~3458866,312283221~8455627,322680407~1513294,329448555~32850,340503058~12373579,353208019~14153502,369311906~8107396,378273435~10682903,397672230~21106695] 2024-03-29T05:08:48.295 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,31236423~6140372,39741677~11631730,56829493~11452450,72073222~3768797,79341797~5110550,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,214230671~974716,232982965~7450751,246836530~10875485,263765730~2028814,276609835~9544628,293351984~1296,295753178~3458866,312283221~8455627,322680407~1513294,329448555~32850,340503058~12373579,353208019~14153502,369311906~8107396,378273435~10682903,397672230~21106695] 2024-03-29T05:08:48.301 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,31236423~6140372,38431381~12942026,54525952~13755991,72073222~3768797,79341797~5110550,84790916~6743957,96468992~8117915,104857600~3451536,111013211~305386,111888563~6642772,121634816~5109676,138412032~10719543,161975827~7234472,171966464~26626708,200757268~10915689,213909504~1295883,232982965~7450751,245550311~12161704,260046848~5747696,276609835~9544628,293351984~1296,293991835~5220209,310378496~10360352,322680407~1513294,327155712~2325693,339738624~13138013,353208019~14153502,369311906~8107396,377663290~11293048,394264576~24514349] 2024-03-29T05:08:48.301 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 11 diff [404568~171768,2505262~8011646,13801673~6528238,31236423~6140372,39741677~11631730,56829493~11452450,72073222~3768797,79341797~5110550,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,213342593~7281743,232982965~7450751,246836530~10875485,263765730~2028814,276609835~9544628,293351984~1296,295753178~3458866,312283221~8455627,322513851~1989805,329343811~2701118,340147706~12728931,353208019~14153502,369311906~8107396,378273435~16507122,397672230~21106695] 2024-03-29T05:08:48.301 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,31236423~6140372,39741677~11631730,56829493~11452450,72073222~3768797,79341797~5110550,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,213342593~7281743,232982965~7450751,246836530~10875485,263765730~2028814,276609835~9544628,293351984~1296,295753178~3458866,312283221~8455627,322513851~1989805,329343811~2701118,340147706~12728931,353208019~14153502,369311906~8107396,378273435~16507122,397672230~21106695] 2024-03-29T05:08:48.306 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,31236423~6140372,38431381~12942026,54525952~13755991,72073222~3768797,79341797~5110550,84790916~6743957,96468992~8117915,104857600~3451536,111013211~305386,111888563~6642772,121634816~5109676,138412032~13132147,161975827~7234472,171966464~26626708,200757268~19867068,232982965~7450751,245550311~12161704,260046848~5747696,276609835~9544628,293351984~1296,293991835~5220209,310378496~10360352,322513851~1989805,327155712~4889217,339738624~13138013,353208019~14153502,369311906~8107396,377663290~41115635] 2024-03-29T05:08:48.306 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 12 diff [404568~171768,2505262~8011646,13801673~6528238,28709632~1902975,31236423~6140372,39741677~11631730,56829493~11452450,72073222~3768797,78839889~5612458,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,161975827~3924913,167413531~1796768,173109571~27176782,200757268~10915689,213342593~7281743,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,340147706~12728931,353208019~14153502,369311906~8107396,378273435~16507122,397672230~21106695] 2024-03-29T05:08:48.306 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,28709632~1902975,31236423~6140372,39741677~11631730,56829493~11452450,72073222~3768797,78839889~5612458,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,161975827~3924913,167413531~1796768,173109571~27176782,200757268~10915689,213342593~7281743,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,340147706~12728931,353208019~14153502,369311906~8107396,378273435~16507122,397672230~21106695] 2024-03-29T05:08:48.312 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,25165824~5446783,31236423~6140372,38431381~12942026,54525952~13755991,72073222~3768797,78839889~5612458,84790916~6743957,96468992~8117915,104857600~3451536,111013211~305386,111888563~6642772,121634816~5109676,138412032~13132147,161975827~7234472,171966464~28319889,200757268~19867068,232982965~7450751,245550311~12161704,260046848~5747696,274173672~11980791,293351984~1296,293991835~5220209,306184192~14554656,322513851~1989805,327155712~4889217,339738624~13138013,353208019~14153502,369311906~8107396,377663290~41115635] 2024-03-29T05:08:48.312 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 13 diff [404568~171768,2505262~8011646,13801673~10588900,28709632~1902975,31236423~6140372,39741677~11631730,56829493~11697280,72073222~3768797,78839889~5612458,86382691~5152182,98583688~6003219,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~27176782,200757268~10915689,213342593~7281743,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~16076441,353208019~24211283,378273435~40505490] 2024-03-29T05:08:48.312 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~10588900,28709632~1902975,31236423~6140372,39741677~11631730,56829493~11697280,72073222~3768797,78839889~5612458,86382691~5152182,98583688~6003219,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~27176782,200757268~10915689,213342593~7281743,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~16076441,353208019~24211283,378273435~40505490] 2024-03-29T05:08:48.317 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~5446783,31236423~6140372,38431381~12942026,54525952~14000821,72073222~3768797,78839889~5612458,84790916~6743957,96468992~8117915,104857600~3451536,109482434~9048901,121634816~13523600,138412032~13132147,161975827~7234472,171966464~28319889,200757268~19867068,232982965~7450751,245550311~12161704,260046848~5747696,274173672~11980791,293351984~1296,293991835~5220209,306184192~14554656,322513851~1989805,327155712~4889217,335544320~17332317,353208019~24211283,377663290~41115635] 2024-03-29T05:08:48.317 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 14 diff [404568~171768,2505262~8011646,13801673~10588900,27077192~10299603,39741677~11631730,56829493~11697280,72073222~3768797,78839889~5612458,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.317 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~10588900,27077192~10299603,39741677~11631730,56829493~11697280,72073222~3768797,78839889~5612458,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.323 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12210971,38431381~12942026,54525952~14000821,72073222~3768797,78839889~5612458,84790916~6743957,92274688~12312219,104857600~3451536,109482434~9048901,121634816~13523600,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~12161704,260046848~5747696,274173672~11980791,293351984~1296,293991835~5220209,306184192~14554656,322513851~1989805,327155712~4889217,335544320~83234605] 2024-03-29T05:08:48.323 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 15 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,56829493~11697280,72073222~3768797,78839889~5612458,84964601~734981,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.323 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,56829493~11697280,72073222~3768797,78839889~5612458,84964601~734981,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.328 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~13563782,54525952~14000821,72073222~3768797,78839889~5612458,84790916~6743957,92274688~12312219,104857600~3451536,109482434~9048901,121634816~13523600,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~12161704,260046848~5747696,274173672~11980791,293351984~1296,293991835~5220209,306184192~14554656,322513851~1989805,327155712~4889217,335544320~83234605] 2024-03-29T05:08:48.328 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 16 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,56829493~11697280,72073222~3768797,78839889~12694984,93785211~10801696,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.329 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,56829493~11697280,72073222~3768797,78839889~12694984,93785211~10801696,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336800196~81978729] 2024-03-29T05:08:48.334 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78839889~12694984,92274688~12312219,104857600~4221929,109482434~10542390,121634816~13523600,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~12161704,260046848~5747696,274173672~11980791,293351984~1296,293991835~5220209,306184192~14554656,322513851~9531078,335544320~83234605] 2024-03-29T05:08:48.334 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 17 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,56829493~11697280,72073222~3768797,78839889~25747018,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~16887296,293351984~1296,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336122526~82656399] 2024-03-29T05:08:48.334 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,56829493~11697280,72073222~3768797,78839889~25747018,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~16887296,293351984~1296,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336122526~82656399] 2024-03-29T05:08:48.340 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78839889~25747018,104857600~4221929,109482434~10542390,121634816~13523600,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~17719007,293351984~1296,293991835~5220209,306184192~14554656,322513851~9531078,335544320~83234605] 2024-03-29T05:08:48.340 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 18 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~16887296,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~5127042,329343811~2701118,333824662~117846,336122526~82656399] 2024-03-29T05:08:48.340 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~16887296,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~5127042,329343811~2701118,333824662~117846,336122526~82656399] 2024-03-29T05:08:48.345 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~17719007,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~11428657,335544320~83234605] 2024-03-29T05:08:48.346 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 19 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:48.346 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:48.351 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:08:48.351 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 4 diff [31236423~4602591,66224838~1583868,79341797~5078800,86382691~5152182,167413531~1796768,173109571~9193179,232982965~7450751,276609835~7456442,369627346~5540348] 2024-03-29T05:08:48.352 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [31236423~4602591,66224838~1583868,79341797~5078800,86382691~5152182,167413531~1796768,173109571~9193179,232982965~7450751,276609835~7456442,369627346~5540348] 2024-03-29T05:08:48.358 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [31236423~4602591,66224838~1583868,79341797~5078800,84790916~6743957,164536940~4673359,171966464~10336286,232982965~7450751,276609835~7456442,369627346~5540348] 2024-03-29T05:08:48.358 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 5 diff [31236423~4602591,44602410~5533264,66224838~1583868,79341797~5078800,86382691~5152182,111013211~305386,161975827~3924913,167413531~1796768,173109571~9193179,214230671~974716,232982965~7450751,276609835~9544628,293351984~1296,361795334~5566187,369627346~5540348,378273435~4517086,403089332~1347239] 2024-03-29T05:08:48.358 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [31236423~4602591,44602410~5533264,66224838~1583868,79341797~5078800,86382691~5152182,111013211~305386,161975827~3924913,167413531~1796768,173109571~9193179,214230671~974716,232982965~7450751,276609835~9544628,293351984~1296,361795334~5566187,369627346~5540348,378273435~4517086,403089332~1347239] 2024-03-29T05:08:48.362 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [31236423~4602591,44602410~5533264,66224838~1583868,79341797~5078800,84790916~6743957,111013211~305386,161975827~7234472,171966464~10336286,213909504~1295883,232982965~7450751,276609835~9544628,293351984~1296,361795334~5566187,369627346~5540348,377663290~5127231,403089332~1347239] 2024-03-29T05:08:48.362 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 6 diff [404568~171768,2505262~8011646,31236423~4602591,44602410~5533264,66224838~1583868,79341797~5078800,86382691~5152182,98583688~6003219,111013211~305386,139874896~9256679,161975827~3924913,167413531~1796768,173109571~9193179,214230671~974716,232982965~7450751,247509107~7937298,276609835~9544628,293351984~1296,317479958~508825,351472632~1404005,353208019~2502943,361795334~5566187,369627346~5540348,378273435~4517086,403089332~1347239] 2024-03-29T05:08:48.362 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,31236423~4602591,44602410~5533264,66224838~1583868,79341797~5078800,86382691~5152182,98583688~6003219,111013211~305386,139874896~9256679,161975827~3924913,167413531~1796768,173109571~9193179,214230671~974716,232982965~7450751,247509107~7937298,276609835~9544628,293351984~1296,317479958~508825,351472632~1404005,353208019~2502943,361795334~5566187,369627346~5540348,378273435~4517086,403089332~1347239] 2024-03-29T05:08:48.368 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,31236423~4602591,44602410~5533264,66224838~1583868,79341797~5078800,84790916~6743957,96468992~8117915,111013211~305386,138412032~10719543,161975827~7234472,171966464~10336286,213909504~1295883,232982965~7450751,247509107~7937298,276609835~9544628,293351984~1296,314572800~3415983,351472632~1404005,353208019~2502943,361795334~5566187,369627346~5540348,377663290~5127231,403089332~1347239] 2024-03-29T05:08:48.368 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 7 diff [404568~171768,2505262~8011646,31236423~4602591,42925201~7210473,57010643~8422112,66224838~1583868,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,111013211~305386,139874896~9256679,161975827~3924913,167413531~1796768,173109571~9193179,186720600~9465340,200757268~8102408,214230671~974716,232982965~7450751,247509107~10202908,276609835~9544628,293351984~1296,312283221~8455627,351472632~1404005,353208019~2502943,361795334~5566187,369627346~5540348,378273435~4517086,403089332~1347239] 2024-03-29T05:08:48.368 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,31236423~4602591,42925201~7210473,57010643~8422112,66224838~1583868,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,111013211~305386,139874896~9256679,161975827~3924913,167413531~1796768,173109571~9193179,186720600~9465340,200757268~8102408,214230671~974716,232982965~7450751,247509107~10202908,276609835~9544628,293351984~1296,312283221~8455627,351472632~1404005,353208019~2502943,361795334~5566187,369627346~5540348,378273435~4517086,403089332~1347239] 2024-03-29T05:08:48.373 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,31236423~4602591,42925201~7210473,57010643~8422112,66224838~1583868,72709431~3132588,79341797~5078800,84790916~6743957,96468992~8117915,111013211~305386,138412032~10719543,161975827~7234472,171966464~10336286,186720600~9465340,200757268~8102408,213909504~1295883,232982965~7450751,247509107~10202908,276609835~9544628,293351984~1296,310378496~10360352,351472632~1404005,353208019~2502943,361795334~5566187,369627346~5540348,377663290~5127231,403089332~1347239] 2024-03-29T05:08:48.373 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 8 diff [404568~171768,2505262~8011646,13801673~6528238,31236423~4602591,42925201~8448206,57010643~11271300,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~23076369,200757268~8102408,214230671~974716,232982965~7450751,246836530~10875485,276609835~9544628,293351984~1296,312283221~8455627,351472632~1404005,353208019~2502943,361795334~5566187,369627346~5540348,378273435~4517086,401507631~5667119,411204045~7574880] 2024-03-29T05:08:48.373 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,31236423~4602591,42925201~8448206,57010643~11271300,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~23076369,200757268~8102408,214230671~974716,232982965~7450751,246836530~10875485,276609835~9544628,293351984~1296,312283221~8455627,351472632~1404005,353208019~2502943,361795334~5566187,369627346~5540348,378273435~4517086,401507631~5667119,411204045~7574880] 2024-03-29T05:08:48.378 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,31236423~4602591,42925201~8448206,57010643~11271300,72709431~3132588,79341797~5078800,84790916~6743957,96468992~8117915,104857600~3451536,111013211~305386,121634816~5109676,138412032~10719543,161975827~7234472,171966464~24219476,200757268~8102408,213909504~1295883,232982965~7450751,245550311~12161704,276609835~9544628,293351984~1296,310378496~10360352,351472632~1404005,353208019~2502943,361795334~5566187,369627346~5540348,377663290~5127231,398458880~8715870,411204045~7574880] 2024-03-29T05:08:48.379 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 9 diff [404568~171768,2505262~8011646,13801673~6528238,31236423~4602591,39741677~11631730,57010643~11271300,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,214230671~974716,232982965~7450751,246836530~10875485,263765730~2028814,276609835~9544628,293351984~1296,312283221~8455627,351472632~1404005,353208019~2502943,361795334~5566187,369627346~5540348,378273435~10682903,397672230~9502520,411204045~7574880] 2024-03-29T05:08:48.379 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,31236423~4602591,39741677~11631730,57010643~11271300,74221867~1620152,79341797~5078800,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,214230671~974716,232982965~7450751,246836530~10875485,263765730~2028814,276609835~9544628,293351984~1296,312283221~8455627,351472632~1404005,353208019~2502943,361795334~5566187,369627346~5540348,378273435~10682903,397672230~9502520,411204045~7574880] 2024-03-29T05:08:48.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:47 smithi187 ceph-mon[93536]: pgmap v1048: 105 pgs: 105 active+clean; 696 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 226 KiB/s rd, 10 MiB/s wr, 338 op/s 2024-03-29T05:08:48.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:47 smithi187 ceph-mon[93536]: osdmap e881: 8 total, 8 up, 8 in 2024-03-29T05:08:48.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:47 smithi187 ceph-mon[93536]: failed to encode map e881 with expected crc 2024-03-29T05:08:48.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:47 smithi187 ceph-mon[93536]: failed to encode map e881 with expected crc 2024-03-29T05:08:48.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:47 smithi187 ceph-mon[93536]: failed to encode map e881 with expected crc 2024-03-29T05:08:48.385 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,31236423~4602591,39741677~11631730,57010643~11271300,72709431~3132588,79341797~5078800,84790916~6743957,96468992~8117915,104857600~3451536,111013211~305386,121634816~5109676,138412032~10719543,161975827~7234472,171966464~26626708,200757268~10915689,213909504~1295883,232982965~7450751,245550311~12161704,260046848~5747696,276609835~9544628,293351984~1296,310378496~10360352,351472632~1404005,353208019~2502943,361795334~5566187,369627346~5540348,377663290~11293048,394264576~12910174,411204045~7574880] 2024-03-29T05:08:48.385 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 10 diff [404568~171768,2505262~8011646,13801673~6528238,31236423~6140372,39741677~11631730,57010643~11271300,72073222~3768797,79341797~5110550,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,214230671~974716,232982965~7450751,246836530~10875485,263765730~2028814,276609835~9544628,293351984~1296,295753178~3458866,312283221~8455627,322680407~1513294,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,378273435~10682903,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.385 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,31236423~6140372,39741677~11631730,57010643~11271300,72073222~3768797,79341797~5110550,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~9256679,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,214230671~974716,232982965~7450751,246836530~10875485,263765730~2028814,276609835~9544628,293351984~1296,295753178~3458866,312283221~8455627,322680407~1513294,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,378273435~10682903,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.390 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,31236423~6140372,39741677~11631730,57010643~11271300,72073222~3768797,79341797~5110550,84790916~6743957,96468992~8117915,104857600~3451536,111013211~305386,111888563~6642772,121634816~5109676,138412032~10719543,161975827~7234472,171966464~26626708,200757268~10915689,213909504~1295883,232982965~7450751,245550311~12161704,260046848~5747696,276609835~9544628,293351984~1296,293991835~5220209,310378496~10360352,322680407~1513294,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,377663290~11293048,394264576~12910174,407807916~10971009] 2024-03-29T05:08:48.390 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 11 diff [404568~171768,2505262~8011646,13801673~6528238,31236423~6140372,39741677~11631730,57010643~11271300,72073222~3768797,79341797~5110550,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,213342593~7281743,232982965~7450751,246836530~10875485,263765730~2028814,276609835~9544628,293351984~1296,295753178~3458866,312283221~8455627,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,378273435~16507122,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.390 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,31236423~6140372,39741677~11631730,57010643~11271300,72073222~3768797,79341797~5110550,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,161975827~3924913,167413531~1796768,173109571~25483601,200757268~10915689,213342593~7281743,232982965~7450751,246836530~10875485,263765730~2028814,276609835~9544628,293351984~1296,295753178~3458866,312283221~8455627,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,378273435~16507122,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.395 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,31236423~6140372,39741677~11631730,57010643~11271300,72073222~3768797,79341797~5110550,84790916~6743957,96468992~8117915,104857600~3451536,111013211~305386,111888563~6642772,121634816~5109676,138412032~13132147,161975827~7234472,171966464~26626708,200757268~19867068,232982965~7450751,245550311~12161704,260046848~5747696,276609835~9544628,293351984~1296,293991835~5220209,310378496~10360352,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,377663290~29511460,407807916~10971009] 2024-03-29T05:08:48.395 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 12 diff [404568~171768,2505262~8011646,13801673~6528238,28709632~1902975,31236423~6140372,39741677~11631730,57010643~11271300,72073222~3768797,78839889~5612458,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,161975827~3924913,167413531~1796768,173109571~27176782,200757268~10915689,213342593~7281743,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,378273435~16507122,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.395 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,28709632~1902975,31236423~6140372,39741677~11631730,57010643~11271300,72073222~3768797,78839889~5612458,86382691~5152182,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,161975827~3924913,167413531~1796768,173109571~27176782,200757268~10915689,213342593~7281743,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,378273435~16507122,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.401 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,25165824~5446783,31236423~6140372,39741677~11631730,57010643~11271300,72073222~3768797,78839889~5612458,84790916~6743957,96468992~8117915,104857600~3451536,111013211~305386,111888563~6642772,121634816~5109676,138412032~13132147,161975827~7234472,171966464~28319889,200757268~19867068,232982965~7450751,245550311~12161704,260046848~5747696,274173672~11980791,293351984~1296,293991835~5220209,306184192~14554656,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,377663290~29511460,407807916~10971009] 2024-03-29T05:08:48.401 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 13 diff [404568~171768,2505262~8011646,13801673~10588900,28709632~1902975,31236423~6140372,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,86382691~5152182,98583688~6003219,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~27176782,200757268~10915689,213342593~7281743,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,351472632~1404005,353208019~2502943,361795334~15623968,378273435~28901315,407807916~10971009] 2024-03-29T05:08:48.401 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~10588900,28709632~1902975,31236423~6140372,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,86382691~5152182,98583688~6003219,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~27176782,200757268~10915689,213342593~7281743,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,351472632~1404005,353208019~2502943,361795334~15623968,378273435~28901315,407807916~10971009] 2024-03-29T05:08:48.407 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~5446783,31236423~6140372,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,84790916~6743957,96468992~8117915,104857600~3451536,109482434~9048901,121634816~13523600,138412032~13132147,161975827~7234472,171966464~28319889,200757268~19867068,232982965~7450751,245550311~12161704,260046848~5747696,274173672~11980791,293351984~1296,293991835~5220209,306184192~14554656,322513851~1989805,329343811~2701118,335544320~11223782,351472632~1404005,353208019~2502943,361795334~15623968,377663290~29511460,407807916~10971009] 2024-03-29T05:08:48.407 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 14 diff [404568~171768,2505262~8011646,13801673~10588900,27077192~10299603,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.407 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~10588900,27077192~10299603,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,213342593~7281743,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.413 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12210971,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,84790916~6743957,92274688~12312219,104857600~3451536,109482434~9048901,121634816~13523600,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~12161704,260046848~5747696,274173672~11980791,293351984~1296,293991835~5220209,306184192~14554656,322513851~1989805,329343811~2701118,335544320~11223782,349983031~6849925,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.413 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 15 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,57010643~11516130,72073222~3768797,78839889~5612458,84964601~734981,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.413 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,57010643~11516130,72073222~3768797,78839889~5612458,84964601~734981,86382691~5152182,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.418 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~12253486,57010643~11516130,72073222~3768797,78839889~5612458,84790916~6743957,92274688~12312219,104857600~3451536,109482434~9048901,121634816~13523600,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~12161704,260046848~5747696,274173672~11980791,293351984~1296,293991835~5220209,306184192~14554656,322513851~1989805,329343811~2701118,335544320~11223782,349983031~6849925,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.419 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 16 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57010643~11516130,72073222~3768797,78839889~12694984,93785211~10801696,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336800196~9967906,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.419 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57010643~11516130,72073222~3768797,78839889~12694984,93785211~10801696,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~38563386,212065768~8558568,232982965~7450751,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336800196~9967906,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.424 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78839889~12694984,92274688~12312219,104857600~4221929,109482434~10542390,121634816~13523600,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~12161704,260046848~5747696,274173672~11980791,293351984~1296,293991835~5220209,306184192~14554656,322513851~5127042,329343811~2701118,335544320~11223782,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.424 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 17 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57010643~11516130,72073222~3768797,78839889~25747018,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~16887296,293351984~1296,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.424 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57010643~11516130,72073222~3768797,78839889~25747018,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~16887296,293351984~1296,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.430 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78839889~25747018,104857600~4221929,109482434~10542390,121634816~13523600,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~17719007,293351984~1296,293991835~5220209,306184192~14554656,322513851~5127042,329343811~2701118,335544320~11223782,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.430 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 18 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~16887296,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~5127042,329343811~2701118,333824662~117846,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.430 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~16887296,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~5127042,329343811~2701118,333824662~117846,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.435 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~17719007,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~5127042,329343811~4598697,335544320~11223782,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.436 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 19 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.436 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.441 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.441 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 5 diff [44602410~5533264,111013211~305386,161975827~3924913,173301115~2201551,214230671~974716,282993096~3161367,293351984~1296,361795334~5566187,378273435~4517086,403089332~1347239] 2024-03-29T05:08:48.441 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [44602410~5533264,111013211~305386,161975827~3924913,173301115~2201551,214230671~974716,282993096~3161367,293351984~1296,361795334~5566187,378273435~4517086,403089332~1347239] 2024-03-29T05:08:48.447 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [44602410~5533264,111013211~305386,161975827~3924913,173301115~2201551,213909504~1295883,282993096~3161367,293351984~1296,361795334~5566187,377663290~5127231,403089332~1347239] 2024-03-29T05:08:48.447 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 6 diff [404568~171768,2505262~8011646,44602410~5533264,98583688~6003219,111013211~305386,139874896~9256679,161975827~3924913,173301115~2201551,214230671~974716,247509107~7937298,276882371~9272092,293351984~1296,317479958~508825,351472632~1404005,353208019~2502943,361795334~5566187,378273435~4517086,403089332~1347239] 2024-03-29T05:08:48.447 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,44602410~5533264,98583688~6003219,111013211~305386,139874896~9256679,161975827~3924913,173301115~2201551,214230671~974716,247509107~7937298,276882371~9272092,293351984~1296,317479958~508825,351472632~1404005,353208019~2502943,361795334~5566187,378273435~4517086,403089332~1347239] 2024-03-29T05:08:48.452 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,44602410~5533264,96468992~8117915,111013211~305386,138412032~10719543,161975827~3924913,173301115~2201551,213909504~1295883,247509107~7937298,276882371~9272092,293351984~1296,314572800~3415983,351472632~1404005,353208019~2502943,361795334~5566187,377663290~5127231,403089332~1347239] 2024-03-29T05:08:48.452 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 7 diff [404568~171768,2505262~8011646,42925201~7210473,57010643~8422112,74221867~1620152,98583688~6003219,111013211~305386,139874896~9256679,161975827~3924913,173301115~2201551,186720600~9465340,200757268~8102408,214230671~974716,247509107~10202908,276882371~9272092,293351984~1296,312283221~8455627,351472632~1404005,353208019~2502943,361795334~5566187,378273435~4517086,403089332~1347239] 2024-03-29T05:08:48.452 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,42925201~7210473,57010643~8422112,74221867~1620152,98583688~6003219,111013211~305386,139874896~9256679,161975827~3924913,173301115~2201551,186720600~9465340,200757268~8102408,214230671~974716,247509107~10202908,276882371~9272092,293351984~1296,312283221~8455627,351472632~1404005,353208019~2502943,361795334~5566187,378273435~4517086,403089332~1347239] 2024-03-29T05:08:48.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:47 smithi181 ceph-mon[101970]: pgmap v1048: 105 pgs: 105 active+clean; 696 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 226 KiB/s rd, 10 MiB/s wr, 338 op/s 2024-03-29T05:08:48.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:47 smithi181 ceph-mon[101970]: osdmap e881: 8 total, 8 up, 8 in 2024-03-29T05:08:48.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:47 smithi181 ceph-mon[101970]: failed to encode map e881 with expected crc 2024-03-29T05:08:48.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:47 smithi181 ceph-mon[101970]: failed to encode map e881 with expected crc 2024-03-29T05:08:48.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:47 smithi181 ceph-mon[101970]: failed to encode map e881 with expected crc 2024-03-29T05:08:48.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:47 smithi181 ceph-mon[106466]: pgmap v1048: 105 pgs: 105 active+clean; 696 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 226 KiB/s rd, 10 MiB/s wr, 338 op/s 2024-03-29T05:08:48.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:47 smithi181 ceph-mon[106466]: osdmap e881: 8 total, 8 up, 8 in 2024-03-29T05:08:48.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:47 smithi181 ceph-mon[106466]: failed to encode map e881 with expected crc 2024-03-29T05:08:48.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:47 smithi181 ceph-mon[106466]: failed to encode map e881 with expected crc 2024-03-29T05:08:48.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:47 smithi181 ceph-mon[106466]: failed to encode map e881 with expected crc 2024-03-29T05:08:48.458 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,42925201~7210473,57010643~8422112,72709431~3132588,96468992~8117915,111013211~305386,138412032~10719543,161975827~3924913,173301115~2201551,186720600~9465340,200757268~8102408,213909504~1295883,247509107~10202908,276882371~9272092,293351984~1296,310378496~10360352,351472632~1404005,353208019~2502943,361795334~5566187,377663290~5127231,403089332~1347239] 2024-03-29T05:08:48.458 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 8 diff [404568~171768,2505262~8011646,13801673~6528238,42925201~8448206,57010643~11271300,74221867~1620152,98583688~6003219,106697534~1611602,111013211~305386,124550033~2194459,139874896~9256679,161975827~3924913,173301115~2201551,178154435~2311458,180963130~15222810,200757268~8102408,214230671~974716,246836530~10875485,276882371~9272092,293351984~1296,312283221~8455627,351472632~1404005,353208019~2502943,361795334~5566187,378273435~4517086,401507631~5667119,411204045~7574880] 2024-03-29T05:08:48.458 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,42925201~8448206,57010643~11271300,74221867~1620152,98583688~6003219,106697534~1611602,111013211~305386,124550033~2194459,139874896~9256679,161975827~3924913,173301115~2201551,178154435~2311458,180963130~15222810,200757268~8102408,214230671~974716,246836530~10875485,276882371~9272092,293351984~1296,312283221~8455627,351472632~1404005,353208019~2502943,361795334~5566187,378273435~4517086,401507631~5667119,411204045~7574880] 2024-03-29T05:08:48.463 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,42925201~8448206,57010643~11271300,72709431~3132588,96468992~8117915,104857600~3451536,111013211~305386,121634816~5109676,138412032~10719543,161975827~3924913,173301115~2201551,178154435~2311458,180963130~15222810,200757268~8102408,213909504~1295883,245550311~12161704,276882371~9272092,293351984~1296,310378496~10360352,351472632~1404005,353208019~2502943,361795334~5566187,377663290~5127231,398458880~8715870,411204045~7574880] 2024-03-29T05:08:48.463 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 9 diff [404568~171768,2505262~8011646,13801673~6528238,39741677~11631730,57010643~11271300,74221867~1620152,98583688~6003219,106697534~1611602,111013211~305386,124550033~2194459,139874896~9256679,161975827~3924913,173301115~2201551,178154435~2311458,180963130~17630042,200757268~10915689,214230671~974716,246836530~10875485,263765730~2028814,276882371~9272092,293351984~1296,312283221~8455627,351472632~1404005,353208019~2502943,361795334~5566187,378273435~10682903,397672230~9502520,411204045~7574880] 2024-03-29T05:08:48.463 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,39741677~11631730,57010643~11271300,74221867~1620152,98583688~6003219,106697534~1611602,111013211~305386,124550033~2194459,139874896~9256679,161975827~3924913,173301115~2201551,178154435~2311458,180963130~17630042,200757268~10915689,214230671~974716,246836530~10875485,263765730~2028814,276882371~9272092,293351984~1296,312283221~8455627,351472632~1404005,353208019~2502943,361795334~5566187,378273435~10682903,397672230~9502520,411204045~7574880] 2024-03-29T05:08:48.469 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,39741677~11631730,57010643~11271300,72709431~3132588,96468992~8117915,104857600~3451536,111013211~305386,121634816~5109676,138412032~10719543,161975827~3924913,173301115~2201551,178154435~2311458,180963130~17630042,200757268~10915689,213909504~1295883,245550311~12161704,260046848~5747696,276882371~9272092,293351984~1296,310378496~10360352,351472632~1404005,353208019~2502943,361795334~5566187,377663290~11293048,394264576~12910174,411204045~7574880] 2024-03-29T05:08:48.469 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 10 diff [404568~171768,2505262~8011646,13801673~6528238,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~9256679,161975827~3924913,173301115~2201551,178154435~2311458,180963130~17630042,200757268~10915689,214230671~974716,246836530~10875485,263765730~2028814,276882371~9272092,293351984~1296,295753178~3458866,312283221~8455627,322680407~1513294,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,378273435~10682903,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.469 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~9256679,161975827~3924913,173301115~2201551,178154435~2311458,180963130~17630042,200757268~10915689,214230671~974716,246836530~10875485,263765730~2028814,276882371~9272092,293351984~1296,295753178~3458866,312283221~8455627,322680407~1513294,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,378273435~10682903,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.474 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,96468992~8117915,104857600~3451536,111013211~305386,111888563~6642772,121634816~5109676,138412032~10719543,161975827~3924913,173301115~2201551,178154435~2311458,180963130~17630042,200757268~10915689,213909504~1295883,245550311~12161704,260046848~5747696,276882371~9272092,293351984~1296,293991835~5220209,310378496~10360352,322680407~1513294,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,377663290~11293048,394264576~12910174,407807916~10971009] 2024-03-29T05:08:48.474 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 11 diff [404568~171768,2505262~8011646,13801673~6528238,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,161975827~3924913,173301115~2201551,178154435~2311458,180963130~17630042,200757268~10915689,213342593~7281743,246836530~10875485,263765730~2028814,276882371~9272092,293351984~1296,295753178~3458866,312283221~8455627,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,378273435~16507122,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.474 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,161975827~3924913,173301115~2201551,178154435~2311458,180963130~17630042,200757268~10915689,213342593~7281743,246836530~10875485,263765730~2028814,276882371~9272092,293351984~1296,295753178~3458866,312283221~8455627,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,378273435~16507122,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.480 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,96468992~8117915,104857600~3451536,111013211~305386,111888563~6642772,121634816~5109676,138412032~13132147,161975827~3924913,173301115~2201551,178154435~2311458,180963130~17630042,200757268~19867068,245550311~12161704,260046848~5747696,276882371~9272092,293351984~1296,293991835~5220209,310378496~10360352,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,377663290~29511460,407807916~10971009] 2024-03-29T05:08:48.480 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 12 diff [404568~171768,2505262~8011646,13801673~6528238,28709632~1902975,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,78839889~5612458,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,161975827~3924913,173301115~2201551,178154435~2311458,180963130~19323223,200757268~10915689,213342593~7281743,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,378273435~16507122,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.480 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,28709632~1902975,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,78839889~5612458,98583688~6003219,106697534~1611602,111013211~305386,111888563~6642772,124550033~2194459,139874896~11669283,161975827~3924913,173301115~2201551,178154435~2311458,180963130~19323223,200757268~10915689,213342593~7281743,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,378273435~16507122,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.485 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,25165824~5446783,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,78839889~5612458,96468992~8117915,104857600~3451536,111013211~305386,111888563~6642772,121634816~5109676,138412032~13132147,161975827~3924913,173301115~2201551,178154435~2311458,180963130~19323223,200757268~19867068,245550311~12161704,260046848~5747696,274173672~11980791,293351984~1296,293991835~5220209,306184192~14554656,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,361795334~5566187,369311906~8107396,377663290~29511460,407807916~10971009] 2024-03-29T05:08:48.485 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 13 diff [404568~171768,2505262~8011646,13801673~10588900,28709632~1902975,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,98583688~6003219,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,178154435~2311458,180963130~19323223,200757268~10915689,213342593~7281743,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,351472632~1404005,353208019~2502943,361795334~15623968,378273435~28901315,407807916~10971009] 2024-03-29T05:08:48.485 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~10588900,28709632~1902975,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,98583688~6003219,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,178154435~2311458,180963130~19323223,200757268~10915689,213342593~7281743,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,351472632~1404005,353208019~2502943,361795334~15623968,378273435~28901315,407807916~10971009] 2024-03-29T05:08:48.491 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~5446783,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,96468992~8117915,104857600~3451536,109482434~9048901,121634816~13523600,138412032~13132147,161975827~3924913,173301115~2201551,178154435~2311458,180963130~19323223,200757268~19867068,245550311~12161704,260046848~5747696,274173672~11980791,293351984~1296,293991835~5220209,306184192~14554656,322513851~1989805,329343811~2701118,335544320~11223782,351472632~1404005,353208019~2502943,361795334~15623968,377663290~29511460,407807916~10971009] 2024-03-29T05:08:48.491 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 14 diff [404568~171768,2505262~8011646,13801673~10588900,27077192~6988326,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,178154435~2311458,180963130~30709827,213342593~7281743,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.491 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~10588900,27077192~6988326,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,178154435~2311458,180963130~30709827,213342593~7281743,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.496 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~8899694,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,92274688~12312219,104857600~3451536,109482434~9048901,121634816~13523600,138412032~13132147,161975827~3924913,173301115~2201551,178154435~2311458,180963130~39661206,245550311~12161704,260046848~5747696,274173672~11980791,293351984~1296,293991835~5220209,306184192~14554656,322513851~1989805,329343811~2701118,335544320~11223782,349983031~6849925,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.496 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 15 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,57010643~11516130,72073222~3768797,78839889~5612458,84964601~734981,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,178154435~2311458,180963130~30709827,212065768~8558568,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.496 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,57010643~11516130,72073222~3768797,78839889~5612458,84964601~734981,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,178154435~2311458,180963130~30709827,212065768~8558568,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.502 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~12253486,57010643~11516130,72073222~3768797,78839889~5612458,84964601~734981,92274688~12312219,104857600~3451536,109482434~9048901,121634816~13523600,138412032~13132147,161975827~3924913,173301115~2201551,178154435~2311458,180963130~39661206,245550311~12161704,260046848~5747696,274173672~11980791,293351984~1296,293991835~5220209,306184192~14554656,322513851~1989805,329343811~2701118,335544320~11223782,349983031~6849925,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.502 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 16 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57010643~11516130,72073222~3768797,78839889~9212761,93785211~10801696,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,178154435~2311458,180963130~30709827,212065768~8558568,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336800196~9967906,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.502 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57010643~11516130,72073222~3768797,78839889~9212761,93785211~10801696,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,178154435~2311458,180963130~30709827,212065768~8558568,246836530~10875485,263765730~2028814,274173672~11980791,293351984~1296,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336800196~9967906,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.508 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78839889~9212761,92274688~12312219,104857600~4221929,109482434~10542390,121634816~13523600,138412032~13132147,161975827~3924913,173301115~2201551,178154435~2311458,180963130~39661206,245550311~12161704,260046848~5747696,274173672~11980791,293351984~1296,293991835~5220209,306184192~14554656,322513851~5127042,329343811~2701118,335544320~11223782,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.508 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 17 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57010643~11516130,72073222~3768797,78839889~9212761,88080139~16506768,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~16887296,293351984~1296,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.508 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57010643~11516130,72073222~3768797,78839889~9212761,88080139~16506768,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~16887296,293351984~1296,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.514 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78839889~9212761,88080139~16506768,104857600~4221929,109482434~10542390,121634816~13523600,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~17719007,293351984~1296,293991835~5220209,306184192~14554656,322513851~5127042,329343811~2701118,335544320~11223782,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.514 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 18 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~16887296,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~5127042,329343811~2701118,333824662~117846,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.514 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~16887296,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~5127042,329343811~2701118,333824662~117846,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.519 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~17719007,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~5127042,329343811~4598697,335544320~11223782,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.519 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 19 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.520 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.525 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:48.525 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 6 diff [404568~171768,2505262~8011646,98583688~6003219,139874896~9256679,247509107~7937298,276882371~6602782,317479958~508825,351472632~1404005,353208019~2502943] 2024-03-29T05:08:48.525 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,98583688~6003219,139874896~9256679,247509107~7937298,276882371~6602782,317479958~508825,351472632~1404005,353208019~2502943] 2024-03-29T05:08:48.531 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,96468992~8117915,138412032~10719543,247509107~7937298,276882371~6602782,314572800~3415983,351472632~1404005,353208019~2502943] 2024-03-29T05:08:48.531 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 7 diff [404568~171768,2505262~8011646,42925201~4901136,57010643~8422112,74221867~1620152,98583688~6003219,139874896~9256679,186720600~9465340,200757268~8102408,247509107~10202908,276882371~6602782,312283221~8455627,351472632~1404005,353208019~2502943] 2024-03-29T05:08:48.531 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,42925201~4901136,57010643~8422112,74221867~1620152,98583688~6003219,139874896~9256679,186720600~9465340,200757268~8102408,247509107~10202908,276882371~6602782,312283221~8455627,351472632~1404005,353208019~2502943] 2024-03-29T05:08:48.536 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,42925201~4901136,57010643~8422112,72709431~3132588,96468992~8117915,138412032~10719543,186720600~9465340,200757268~8102408,247509107~10202908,276882371~6602782,310378496~10360352,351472632~1404005,353208019~2502943] 2024-03-29T05:08:48.537 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 8 diff [404568~171768,2505262~8011646,13801673~6528238,42925201~8448206,57010643~11271300,74221867~1620152,98583688~6003219,106697534~1611602,124550033~2194459,139874896~9256679,178154435~2311458,180963130~15222810,200757268~8102408,246836530~10875485,276882371~6602782,312283221~8455627,351472632~1404005,353208019~2502943,401507631~5667119,411204045~7574880] 2024-03-29T05:08:48.537 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,42925201~8448206,57010643~11271300,74221867~1620152,98583688~6003219,106697534~1611602,124550033~2194459,139874896~9256679,178154435~2311458,180963130~15222810,200757268~8102408,246836530~10875485,276882371~6602782,312283221~8455627,351472632~1404005,353208019~2502943,401507631~5667119,411204045~7574880] 2024-03-29T05:08:48.542 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,42925201~8448206,57010643~11271300,72709431~3132588,96468992~8117915,104857600~3451536,121634816~5109676,138412032~10719543,178154435~2311458,180963130~15222810,200757268~8102408,245550311~12161704,276882371~6602782,310378496~10360352,351472632~1404005,353208019~2502943,398458880~8715870,411204045~7574880] 2024-03-29T05:08:48.542 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 9 diff [404568~171768,2505262~8011646,13801673~6528238,39741677~11631730,57010643~11271300,74221867~1620152,98583688~6003219,106697534~1611602,124550033~2194459,139874896~9256679,178154435~2311458,180963130~17630042,200757268~10915689,246836530~10875485,263765730~2028814,276882371~6602782,312283221~8455627,351472632~1404005,353208019~2502943,382357326~6599012,397672230~9502520,411204045~7574880] 2024-03-29T05:08:48.542 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,39741677~11631730,57010643~11271300,74221867~1620152,98583688~6003219,106697534~1611602,124550033~2194459,139874896~9256679,178154435~2311458,180963130~17630042,200757268~10915689,246836530~10875485,263765730~2028814,276882371~6602782,312283221~8455627,351472632~1404005,353208019~2502943,382357326~6599012,397672230~9502520,411204045~7574880] 2024-03-29T05:08:48.547 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,39741677~11631730,57010643~11271300,72709431~3132588,96468992~8117915,104857600~3451536,121634816~5109676,138412032~10719543,178154435~2311458,180963130~17630042,200757268~10915689,245550311~12161704,260046848~5747696,276882371~6602782,310378496~10360352,351472632~1404005,353208019~2502943,382357326~6599012,394264576~12910174,411204045~7574880] 2024-03-29T05:08:48.547 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 10 diff [404568~171768,2505262~8011646,13801673~6528238,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,98583688~6003219,106697534~1611602,111888563~6642772,124550033~2194459,139874896~9256679,178154435~2311458,180963130~17630042,200757268~10915689,246836530~10875485,263765730~2028814,276882371~6602782,295753178~3458866,312283221~8455627,322680407~1513294,351472632~1404005,353208019~2502943,369311906~8107396,382357326~6599012,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.548 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,98583688~6003219,106697534~1611602,111888563~6642772,124550033~2194459,139874896~9256679,178154435~2311458,180963130~17630042,200757268~10915689,246836530~10875485,263765730~2028814,276882371~6602782,295753178~3458866,312283221~8455627,322680407~1513294,351472632~1404005,353208019~2502943,369311906~8107396,382357326~6599012,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.553 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,96468992~8117915,104857600~3451536,111888563~6642772,121634816~5109676,138412032~10719543,178154435~2311458,180963130~17630042,200757268~10915689,245550311~12161704,260046848~5747696,276882371~6602782,293991835~5220209,310378496~10360352,322680407~1513294,351472632~1404005,353208019~2502943,369311906~8107396,382357326~6599012,394264576~12910174,407807916~10971009] 2024-03-29T05:08:48.553 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 11 diff [404568~171768,2505262~8011646,13801673~6528238,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,98583688~6003219,106697534~1611602,111888563~6642772,124550033~2194459,139874896~11669283,178154435~2311458,180963130~17630042,200757268~10915689,213342593~7281743,246836530~10875485,263765730~2028814,276882371~6602782,295753178~3458866,312283221~8455627,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,366005835~1027758,369311906~8107396,382357326~12423231,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.553 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,98583688~6003219,106697534~1611602,111888563~6642772,124550033~2194459,139874896~11669283,178154435~2311458,180963130~17630042,200757268~10915689,213342593~7281743,246836530~10875485,263765730~2028814,276882371~6602782,295753178~3458866,312283221~8455627,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,366005835~1027758,369311906~8107396,382357326~12423231,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.558 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,96468992~8117915,104857600~3451536,111888563~6642772,121634816~5109676,138412032~13132147,178154435~2311458,180963130~17630042,200757268~19867068,245550311~12161704,260046848~5747696,276882371~6602782,293991835~5220209,310378496~10360352,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,366005835~1027758,369311906~8107396,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.558 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 12 diff [404568~171768,2505262~8011646,13801673~6528238,28709632~1902975,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,78839889~5612458,98583688~6003219,106697534~1611602,111888563~6642772,124550033~2194459,139874896~11669283,178154435~2311458,180963130~19323223,200757268~10915689,213342593~7281743,246836530~10875485,263765730~2028814,274173672~9311481,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,366005835~1027758,369311906~8107396,382357326~12423231,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.558 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~6528238,28709632~1902975,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,78839889~5612458,98583688~6003219,106697534~1611602,111888563~6642772,124550033~2194459,139874896~11669283,178154435~2311458,180963130~19323223,200757268~10915689,213342593~7281743,246836530~10875485,263765730~2028814,274173672~9311481,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,366005835~1027758,369311906~8107396,382357326~12423231,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.564 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~7746999,25165824~5446783,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,78839889~5612458,96468992~8117915,104857600~3451536,111888563~6642772,121634816~5109676,138412032~13132147,178154435~2311458,180963130~19323223,200757268~19867068,245550311~12161704,260046848~5747696,274173672~9311481,293991835~5220209,306184192~14554656,322513851~1989805,329343811~2701118,340147706~6620396,351472632~1404005,353208019~2502943,366005835~1027758,369311906~8107396,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.564 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 13 diff [404568~171768,2505262~8011646,13801673~10588900,28709632~1902975,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,98583688~6003219,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,178154435~2311458,180963130~19323223,200757268~10915689,213342593~7281743,246836530~10875485,263765730~2028814,274173672~9311481,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,351472632~1404005,353208019~2502943,365857476~11561826,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.564 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~10588900,28709632~1902975,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,98583688~6003219,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,178154435~2311458,180963130~19323223,200757268~10915689,213342593~7281743,246836530~10875485,263765730~2028814,274173672~9311481,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,351472632~1404005,353208019~2502943,365857476~11561826,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.569 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~5446783,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,96468992~8117915,104857600~3451536,109482434~9048901,121634816~13523600,138412032~13132147,178154435~2311458,180963130~19323223,200757268~19867068,245550311~12161704,260046848~5747696,274173672~9311481,293991835~5220209,306184192~14554656,322513851~1989805,329343811~2701118,335544320~11223782,351472632~1404005,353208019~2502943,365857476~11561826,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.570 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 14 diff [404568~171768,2505262~8011646,13801673~10588900,27077192~6988326,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,178154435~2311458,180963130~30709827,213342593~7281743,246836530~10875485,263765730~2028814,274173672~9311481,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.570 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,2505262~8011646,13801673~10588900,27077192~6988326,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,178154435~2311458,180963130~30709827,213342593~7281743,246836530~10875485,263765730~2028814,274173672~9311481,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.575 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~8899694,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,92274688~12312219,104857600~3451536,109482434~9048901,121634816~13523600,138412032~13132147,178154435~2311458,180963130~39661206,245550311~12161704,260046848~5747696,274173672~9311481,293991835~5220209,306184192~14554656,322513851~1989805,329343811~2701118,335544320~11223782,349983031~6849925,365857476~12524908,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.575 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 15 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,57010643~11516130,72073222~3768797,78839889~5612458,84964601~734981,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,178154435~2311458,180963130~30709827,212065768~8558568,246836530~10875485,263765730~2028814,274173672~9311481,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,381994856~25179894,407807916~10971009] 2024-03-29T05:08:48.575 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,57010643~11516130,72073222~3768797,78839889~5612458,84964601~734981,93785211~10801696,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,139874896~11669283,178154435~2311458,180963130~30709827,212065768~8558568,246836530~10875485,263765730~2028814,274173672~9311481,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,381994856~25179894,407807916~10971009] 2024-03-29T05:08:48.581 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~12253486,57010643~11516130,72073222~3768797,78839889~5612458,84964601~734981,92274688~12312219,104857600~3451536,109482434~9048901,121634816~13523600,138412032~13132147,178154435~2311458,180963130~39661206,245550311~12161704,260046848~5747696,274173672~9311481,293991835~5220209,306184192~14554656,322513851~1989805,329343811~2701118,335544320~11223782,349983031~6849925,365857476~12524908,381994856~25179894,407807916~10971009] 2024-03-29T05:08:48.581 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 16 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57010643~11516130,72073222~3768797,78839889~9212761,93785211~10801696,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,178154435~2311458,180963130~30709827,212065768~8558568,246836530~10875485,263765730~2028814,274173672~9311481,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336800196~9967906,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.581 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57010643~11516130,72073222~3768797,78839889~9212761,93785211~10801696,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,178154435~2311458,180963130~30709827,212065768~8558568,246836530~10875485,263765730~2028814,274173672~9311481,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336800196~9967906,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.586 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78839889~9212761,92274688~12312219,104857600~4221929,109482434~10542390,121634816~13523600,138412032~13132147,178154435~2311458,180963130~39661206,245550311~12161704,260046848~5747696,274173672~9311481,293991835~5220209,306184192~14554656,322513851~5127042,329343811~2701118,335544320~11223782,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.586 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 17 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57010643~11516130,72073222~3768797,78839889~9212761,88080139~16506768,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.587 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57010643~11516130,72073222~3768797,78839889~9212761,88080139~16506768,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.592 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78839889~9212761,88080139~16506768,104857600~4221929,109482434~10542390,121634816~13523600,138412032~13132147,176445827~44178509,245550311~20244233,268435456~15049697,293991835~5220209,306184192~14554656,322513851~5127042,329343811~2701118,335544320~11223782,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.592 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 18 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,295753178~5547167,304147414~1639746,307919109~12819739,322513851~5127042,329343811~2701118,333824662~117846,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.592 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,295753178~5547167,304147414~1639746,307919109~12819739,322513851~5127042,329343811~2701118,333824662~117846,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.597 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,176445827~44178509,245550311~20244233,268435456~15049697,293991835~7308510,301989888~3797272,306184192~14554656,322513851~5127042,329343811~4598697,335544320~11223782,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.597 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 19 diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.597 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.603 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,176445827~44178509,245550311~20244233,268435456~15049697,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.603 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 7 diff [42925201~4901136,57010643~8422112,74221867~1620152,186720600~9465340,200757268~8102408,253041056~4670959,312283221~8455627] 2024-03-29T05:08:48.603 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [42925201~4901136,57010643~8422112,74221867~1620152,186720600~9465340,200757268~8102408,253041056~4670959,312283221~8455627] 2024-03-29T05:08:48.608 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [42925201~4901136,57010643~8422112,72709431~3132588,186720600~9465340,200757268~8102408,253041056~4670959,310378496~10360352] 2024-03-29T05:08:48.609 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 8 diff [13801673~6528238,42925201~8448206,57010643~11271300,74221867~1620152,106697534~1611602,124550033~2194459,178154435~2311458,180963130~15222810,200757268~8102408,246836530~4250329,253041056~4670959,312283221~8455627,401507631~5667119,411204045~7574880] 2024-03-29T05:08:48.609 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [13801673~6528238,42925201~8448206,57010643~11271300,74221867~1620152,106697534~1611602,124550033~2194459,178154435~2311458,180963130~15222810,200757268~8102408,246836530~4250329,253041056~4670959,312283221~8455627,401507631~5667119,411204045~7574880] 2024-03-29T05:08:48.614 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [12582912~7746999,42925201~8448206,57010643~11271300,72709431~3132588,104857600~3451536,121634816~5109676,178154435~2311458,180963130~15222810,200757268~8102408,245550311~5536548,253041056~4670959,310378496~10360352,398458880~8715870,411204045~7574880] 2024-03-29T05:08:48.614 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 9 diff [13801673~6528238,39741677~11631730,57010643~11271300,74221867~1620152,106697534~1611602,124550033~2194459,143026722~1320660,178154435~2311458,180963130~17630042,200757268~10915689,246836530~4250329,253041056~4670959,263765730~2028814,312283221~8455627,382357326~6599012,397672230~9502520,411204045~7574880] 2024-03-29T05:08:48.614 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [13801673~6528238,39741677~11631730,57010643~11271300,74221867~1620152,106697534~1611602,124550033~2194459,143026722~1320660,178154435~2311458,180963130~17630042,200757268~10915689,246836530~4250329,253041056~4670959,263765730~2028814,312283221~8455627,382357326~6599012,397672230~9502520,411204045~7574880] 2024-03-29T05:08:48.619 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [12582912~7746999,39741677~11631730,57010643~11271300,72709431~3132588,104857600~3451536,121634816~5109676,143026722~1320660,178154435~2311458,180963130~17630042,200757268~10915689,245550311~5536548,253041056~4670959,260046848~5747696,310378496~10360352,382357326~6599012,394264576~12910174,411204045~7574880] 2024-03-29T05:08:48.620 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 10 diff [13801673~6528238,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,106697534~1611602,111888563~6642772,124550033~2194459,143026722~1320660,178154435~2311458,180963130~17630042,200757268~10915689,246836530~4250329,253041056~4670959,263765730~2028814,295753178~3458866,312283221~8455627,322680407~1513294,369311906~8107396,382357326~6599012,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.620 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [13801673~6528238,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,106697534~1611602,111888563~6642772,124550033~2194459,143026722~1320660,178154435~2311458,180963130~17630042,200757268~10915689,246836530~4250329,253041056~4670959,263765730~2028814,295753178~3458866,312283221~8455627,322680407~1513294,369311906~8107396,382357326~6599012,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.625 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [12582912~7746999,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,104857600~3451536,111888563~6642772,121634816~5109676,143026722~1320660,178154435~2311458,180963130~17630042,200757268~10915689,245550311~5536548,253041056~4670959,260046848~5747696,293991835~5220209,310378496~10360352,322680407~1513294,369311906~8107396,382357326~6599012,394264576~12910174,407807916~10971009] 2024-03-29T05:08:48.625 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 11 diff [13801673~6528238,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,106697534~1611602,111888563~6642772,124550033~2194459,143026722~1320660,147439418~4104761,178154435~2311458,180963130~17630042,200757268~10915689,213342593~7281743,246836530~4250329,253041056~4670959,263765730~2028814,295753178~3458866,312283221~8455627,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~12423231,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.625 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [13801673~6528238,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,106697534~1611602,111888563~6642772,124550033~2194459,143026722~1320660,147439418~4104761,178154435~2311458,180963130~17630042,200757268~10915689,213342593~7281743,246836530~4250329,253041056~4670959,263765730~2028814,295753178~3458866,312283221~8455627,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~12423231,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.631 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [12582912~7746999,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,81352804~3099543,104857600~3451536,111888563~6642772,121634816~5109676,143026722~1320660,147439418~4104761,178154435~2311458,180963130~17630042,200757268~19867068,245550311~5536548,253041056~4670959,260046848~5747696,293991835~5220209,310378496~10360352,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.631 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 12 diff [13801673~6528238,28709632~1902975,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,78839889~5612458,106697534~1611602,111888563~6642772,124550033~2194459,143026722~1320660,147439418~4104761,178154435~2311458,180963130~19323223,200757268~10915689,213342593~7281743,246836530~4250329,253041056~4670959,263765730~2028814,274173672~5897142,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~12423231,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.631 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [13801673~6528238,28709632~1902975,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,78839889~5612458,106697534~1611602,111888563~6642772,124550033~2194459,143026722~1320660,147439418~4104761,178154435~2311458,180963130~19323223,200757268~10915689,213342593~7281743,246836530~4250329,253041056~4670959,263765730~2028814,274173672~5897142,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~12423231,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.636 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [12582912~7746999,25165824~5446783,34926731~2450064,39741677~11631730,57010643~11271300,72073222~3768797,78839889~5612458,104857600~3451536,111888563~6642772,121634816~5109676,143026722~1320660,147439418~4104761,178154435~2311458,180963130~19323223,200757268~19867068,245550311~5536548,253041056~4670959,260046848~5747696,274173672~5897142,293991835~5220209,306184192~14554656,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.637 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 13 diff [13801673~10588900,28709632~1902975,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,143026722~1320660,147439418~4104761,178154435~2311458,180963130~19323223,200757268~10915689,213342593~7281743,246836530~4250329,253041056~4670959,263765730~2028814,274173672~5897142,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,365857476~11561826,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.637 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [13801673~10588900,28709632~1902975,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,143026722~1320660,147439418~4104761,178154435~2311458,180963130~19323223,200757268~10915689,213342593~7281743,246836530~4250329,253041056~4670959,263765730~2028814,274173672~5897142,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,365857476~11561826,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.642 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [12582912~11807661,25165824~5446783,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,104857600~3451536,109482434~9048901,121634816~13523600,143026722~1320660,147439418~4104761,178154435~2311458,180963130~19323223,200757268~19867068,245550311~5536548,253041056~4670959,260046848~5747696,274173672~5897142,293991835~5220209,306184192~14554656,322513851~1989805,329343811~2701118,335544320~11223782,365857476~11561826,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.642 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 14 diff [13801673~10588900,27077192~6988326,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,93785211~7481883,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,143026722~4018474,147439418~4104761,178154435~2311458,180963130~30709827,213342593~7281743,246836530~4250329,253041056~4670959,263765730~2028814,274173672~5897142,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.643 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [13801673~10588900,27077192~6988326,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,93785211~7481883,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,143026722~4018474,147439418~4104761,178154435~2311458,180963130~30709827,213342593~7281743,246836530~4250329,253041056~4670959,263765730~2028814,274173672~5897142,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.648 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [12582912~11807661,25165824~8899694,34926731~2450064,39741677~11631730,57010643~11516130,72073222~3768797,78839889~5612458,92274688~8992406,104857600~3451536,109482434~9048901,121634816~13523600,143026722~4018474,147439418~4104761,178154435~2311458,180963130~39661206,245550311~5536548,253041056~4670959,260046848~5747696,274173672~5897142,293991835~5220209,306184192~14554656,322513851~1989805,329343811~2701118,335544320~11223782,349983031~6849925,365857476~12524908,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.648 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 15 diff [607349~4280051,13801673~10588900,27077192~10396580,39741677~12253486,57010643~11516130,72073222~3768797,78839889~5612458,84964601~734981,93785211~10560945,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,143026722~4018474,147439418~4104761,178154435~2311458,180963130~30709827,212065768~8558568,246836530~4250329,253041056~4670959,263765730~2028814,274173672~5897142,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,381994856~25179894,407807916~10971009] 2024-03-29T05:08:48.648 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~4280051,13801673~10588900,27077192~10396580,39741677~12253486,57010643~11516130,72073222~3768797,78839889~5612458,84964601~734981,93785211~10560945,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,143026722~4018474,147439418~4104761,178154435~2311458,180963130~30709827,212065768~8558568,246836530~4250329,253041056~4670959,263765730~2028814,274173672~5897142,295753178~3458866,307919109~12819739,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,381994856~25179894,407807916~10971009] 2024-03-29T05:08:48.654 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~4280051,12582912~11807661,25165824~12307948,39741677~12253486,57010643~11516130,72073222~3768797,78839889~5612458,84964601~734981,92274688~12071468,104857600~3451536,109482434~9048901,121634816~13523600,143026722~4018474,147439418~4104761,178154435~2311458,180963130~39661206,245550311~5536548,253041056~4670959,260046848~5747696,274173672~5897142,293991835~5220209,306184192~14554656,322513851~1989805,329343811~2701118,335544320~11223782,349983031~6849925,365857476~12524908,381994856~25179894,407807916~10971009] 2024-03-29T05:08:48.654 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 16 diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57010643~11516130,72073222~3768797,78839889~9212761,93785211~10560945,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,143026722~4018474,147439418~4104761,178154435~2311458,180963130~30709827,212065768~8558568,246836530~4250329,253041056~4670959,263765730~2028814,274173672~5897142,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336800196~9967906,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.654 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57010643~11516130,72073222~3768797,78839889~9212761,93785211~10560945,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,143026722~4018474,147439418~4104761,178154435~2311458,180963130~30709827,212065768~8558568,246836530~4250329,253041056~4670959,263765730~2028814,274173672~5897142,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336800196~9967906,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.660 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78839889~9212761,92274688~12071468,104857600~4221929,109482434~10542390,121634816~13523600,143026722~4018474,147439418~4104761,178154435~2311458,180963130~39661206,245550311~5536548,253041056~4670959,260046848~5747696,274173672~5897142,293991835~5220209,306184192~14554656,322513851~5127042,329343811~2701118,335544320~11223782,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.660 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 17 diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57010643~11516130,72073222~3768797,78839889~9212761,88080139~16266017,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,253041056~12753488,269267167~10803647,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.660 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57010643~11516130,72073222~3768797,78839889~9212761,88080139~16266017,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,253041056~12753488,269267167~10803647,295753178~3458866,307919109~12819739,322513851~5127042,329343811~2701118,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.666 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78839889~9212761,88080139~16266017,104857600~4221929,109482434~10542390,121634816~13523600,143026722~4018474,147439418~4104761,176445827~44178509,245550311~5536548,253041056~12753488,268435456~11635358,293991835~5220209,306184192~14554656,322513851~5127042,329343811~2701118,335544320~11223782,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.666 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 18 diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,253041056~12753488,269267167~10803647,295753178~5547167,304147414~1639746,307919109~12819739,322513851~5127042,329343811~2701118,333824662~117846,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.666 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,253041056~12753488,269267167~10803647,295753178~5547167,304147414~1639746,307919109~12819739,322513851~5127042,329343811~2701118,333824662~117846,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.671 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~44178509,245550311~5536548,253041056~12753488,268435456~11635358,293991835~7308510,301989888~3797272,306184192~14554656,322513851~5127042,329343811~4598697,335544320~11223782,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.672 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 19 diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.672 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.678 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~44178509,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.678 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 8 diff [13801673~6528238,47224125~4149282,65183907~3098036,106697534~1611602,124550033~2194459,178154435~2311458,180963130~7391677,246836530~4250329,401507631~5667119,411204045~7574880] 2024-03-29T05:08:48.678 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [13801673~6528238,47224125~4149282,65183907~3098036,106697534~1611602,124550033~2194459,178154435~2311458,180963130~7391677,246836530~4250329,401507631~5667119,411204045~7574880] 2024-03-29T05:08:48.683 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [12582912~7746999,47224125~4149282,65183907~3098036,104857600~3451536,121634816~5109676,178154435~2311458,180963130~7391677,245550311~5536548,398458880~8715870,411204045~7574880] 2024-03-29T05:08:48.683 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 9 diff [13801673~6528238,39741677~6065998,47096226~4277181,65183907~3098036,106697534~1611602,124550033~2194459,143026722~1320660,178154435~2311458,180963130~7391677,192736931~5856241,207668912~4004045,246836530~4250329,263765730~2028814,382357326~6599012,397672230~9502520,411204045~7574880] 2024-03-29T05:08:48.683 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [13801673~6528238,39741677~6065998,47096226~4277181,65183907~3098036,106697534~1611602,124550033~2194459,143026722~1320660,178154435~2311458,180963130~7391677,192736931~5856241,207668912~4004045,246836530~4250329,263765730~2028814,382357326~6599012,397672230~9502520,411204045~7574880] 2024-03-29T05:08:48.689 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [12582912~7746999,39741677~6065998,47096226~4277181,65183907~3098036,104857600~3451536,121634816~5109676,143026722~1320660,178154435~2311458,180963130~7391677,192736931~5856241,207668912~4004045,245550311~5536548,260046848~5747696,382357326~6599012,394264576~12910174,411204045~7574880] 2024-03-29T05:08:48.689 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 10 diff [13801673~6528238,34926731~2450064,39741677~11631730,65183907~3098036,72073222~2361176,81352804~3099543,106697534~1611602,111888563~6642772,124550033~2194459,143026722~1320660,178154435~2311458,180963130~7391677,192736931~5856241,200988779~4276349,207668912~4004045,246836530~4250329,263765730~2028814,295753178~3458866,322680407~1513294,369311906~8107396,382357326~6599012,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.689 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [13801673~6528238,34926731~2450064,39741677~11631730,65183907~3098036,72073222~2361176,81352804~3099543,106697534~1611602,111888563~6642772,124550033~2194459,143026722~1320660,178154435~2311458,180963130~7391677,192736931~5856241,200988779~4276349,207668912~4004045,246836530~4250329,263765730~2028814,295753178~3458866,322680407~1513294,369311906~8107396,382357326~6599012,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.694 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [12582912~7746999,34926731~2450064,39741677~11631730,65183907~3098036,72073222~2361176,81352804~3099543,104857600~3451536,111888563~6642772,121634816~5109676,143026722~1320660,178154435~2311458,180963130~7391677,192736931~5856241,200988779~4276349,207668912~4004045,245550311~5536548,260046848~5747696,293991835~5220209,322680407~1513294,369311906~8107396,382357326~6599012,394264576~12910174,407807916~10971009] 2024-03-29T05:08:48.694 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 11 diff [13801673~6528238,34926731~2450064,39741677~11631730,59139709~4018447,65183907~3098036,72073222~2361176,81352804~3099543,106697534~1611602,111888563~6642772,124550033~2194459,143026722~1320660,147439418~4104761,178154435~2311458,180963130~7391677,189795676~8797496,200988779~4276349,207668912~4004045,213342593~7281743,246836530~4250329,263765730~2028814,295753178~3458866,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~12423231,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.694 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [13801673~6528238,34926731~2450064,39741677~11631730,59139709~4018447,65183907~3098036,72073222~2361176,81352804~3099543,106697534~1611602,111888563~6642772,124550033~2194459,143026722~1320660,147439418~4104761,178154435~2311458,180963130~7391677,189795676~8797496,200988779~4276349,207668912~4004045,213342593~7281743,246836530~4250329,263765730~2028814,295753178~3458866,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~12423231,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.700 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [12582912~7746999,34926731~2450064,39741677~11631730,59139709~4018447,65183907~3098036,72073222~2361176,81352804~3099543,104857600~3451536,111888563~6642772,121634816~5109676,143026722~1320660,147439418~4104761,178154435~2311458,180963130~7391677,189795676~8797496,200988779~4276349,207668912~12955424,245550311~5536548,260046848~5747696,293991835~5220209,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.700 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 12 diff [13801673~6528238,28709632~1902975,34926731~2450064,39741677~11631730,59139709~4018447,65183907~3098036,72073222~2361176,78839889~5612458,106697534~1611602,111888563~6642772,124550033~2194459,143026722~1320660,147439418~4104761,178154435~2311458,180963130~7391677,189795676~10490677,200988779~4276349,207668912~4004045,213342593~7281743,246836530~4250329,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~12423231,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.700 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [13801673~6528238,28709632~1902975,34926731~2450064,39741677~11631730,59139709~4018447,65183907~3098036,72073222~2361176,78839889~5612458,106697534~1611602,111888563~6642772,124550033~2194459,143026722~1320660,147439418~4104761,178154435~2311458,180963130~7391677,189795676~10490677,200988779~4276349,207668912~4004045,213342593~7281743,246836530~4250329,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~12423231,397672230~9502520,407807916~10971009] 2024-03-29T05:08:48.706 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [12582912~7746999,25165824~5446783,34926731~2450064,39741677~11631730,59139709~4018447,65183907~3098036,72073222~2361176,78839889~5612458,104857600~3451536,111888563~6642772,121634816~5109676,143026722~1320660,147439418~4104761,178154435~2311458,180963130~7391677,189795676~10490677,200988779~4276349,207668912~12955424,245550311~5536548,254724310~2114882,260046848~5747696,274173672~5897142,293991835~5220209,306184192~11718094,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.706 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 13 diff [13801673~10588900,28709632~1902975,34926731~2450064,39741677~11631730,57640836~6971573,65183907~3342866,72073222~2361176,78839889~5612458,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,143026722~1320660,147439418~4104761,178154435~2311458,180963130~8162978,189795676~10490677,200988779~4276349,207668912~4004045,213342593~7281743,246836530~4250329,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,365857476~11561826,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.706 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [13801673~10588900,28709632~1902975,34926731~2450064,39741677~11631730,57640836~6971573,65183907~3342866,72073222~2361176,78839889~5612458,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,143026722~1320660,147439418~4104761,178154435~2311458,180963130~8162978,189795676~10490677,200988779~4276349,207668912~4004045,213342593~7281743,246836530~4250329,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,365857476~11561826,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.711 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [12582912~11807661,25165824~5446783,34926731~2450064,39741677~11631730,57640836~6971573,65183907~3342866,72073222~2361176,78839889~5612458,104857600~3451536,109482434~9048901,121634816~13523600,143026722~1320660,147439418~4104761,178154435~2311458,180963130~8162978,189795676~10490677,200988779~4276349,207668912~12955424,245550311~5536548,254724310~2114882,260046848~5747696,274173672~5897142,293991835~5220209,306184192~11718094,322513851~1989805,329343811~2701118,335544320~11223782,365857476~11561826,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.711 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 14 diff [13801673~10588900,27077192~6988326,34926731~2450064,39741677~11631730,57640836~6971573,65183907~3342866,72073222~2361176,78839889~5612458,93785211~7481883,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,143026722~4018474,147439418~4104761,178154435~2311458,180963130~8162978,189795676~15469452,207668912~4004045,213342593~7281743,246836530~4250329,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.711 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [13801673~10588900,27077192~6988326,34926731~2450064,39741677~11631730,57640836~6971573,65183907~3342866,72073222~2361176,78839889~5612458,93785211~7481883,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,143026722~4018474,147439418~4104761,178154435~2311458,180963130~8162978,189795676~15469452,207668912~4004045,213342593~7281743,246836530~4250329,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.716 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [12582912~11807661,25165824~8899694,34926731~2450064,39741677~11631730,57640836~6971573,65183907~3342866,72073222~2361176,78839889~5612458,92274688~8992406,104857600~3451536,109482434~9048901,121634816~13523600,143026722~4018474,147439418~4104761,178154435~2311458,180963130~8162978,189795676~15469452,207668912~12955424,245550311~5536548,254724310~2114882,260046848~5747696,274173672~5897142,293991835~5220209,306184192~11718094,322513851~1989805,329343811~2701118,335544320~11223782,349983031~6849925,365857476~12524908,382357326~24817424,407807916~10971009] 2024-03-29T05:08:48.717 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 15 diff [607349~4280051,13801673~10588900,27077192~10396580,39741677~12253486,57640836~6971573,65183907~3342866,72073222~2361176,78839889~5612458,84964601~734981,93785211~10560945,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,143026722~4018474,147439418~4104761,178154435~2311458,180963130~8162978,189795676~16239675,207668912~4004045,212065768~8558568,246836530~4250329,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,381994856~25179894,407807916~10971009] 2024-03-29T05:08:48.717 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~4280051,13801673~10588900,27077192~10396580,39741677~12253486,57640836~6971573,65183907~3342866,72073222~2361176,78839889~5612458,84964601~734981,93785211~10560945,106697534~1611602,109482434~9048901,124550033~2194459,127176832~7981584,143026722~4018474,147439418~4104761,178154435~2311458,180963130~8162978,189795676~16239675,207668912~4004045,212065768~8558568,246836530~4250329,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,381994856~25179894,407807916~10971009] 2024-03-29T05:08:48.723 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~4280051,12582912~11807661,25165824~12307948,39741677~12253486,57640836~6971573,65183907~3342866,72073222~2361176,78839889~5612458,84964601~734981,92274688~12071468,104857600~3451536,109482434~9048901,121634816~13523600,143026722~4018474,147439418~4104761,178154435~2311458,180963130~8162978,189795676~16239675,207668912~12955424,245550311~5536548,254724310~2114882,260046848~5747696,274173672~5897142,293991835~5220209,306184192~11718094,322513851~1989805,329343811~2701118,335544320~11223782,349983031~6849925,365857476~12524908,381994856~25179894,407807916~10971009] 2024-03-29T05:08:48.723 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 16 diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57640836~6971573,65183907~3342866,72073222~2361176,78839889~9212761,93785211~10560945,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,143026722~4018474,147439418~4104761,178154435~2311458,180963130~8162978,189795676~16239675,207668912~4004045,212065768~8558568,246836530~4250329,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~5127042,329343811~2701118,336800196~9967906,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.723 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57640836~6971573,65183907~3342866,72073222~2361176,78839889~9212761,93785211~10560945,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,143026722~4018474,147439418~4104761,178154435~2311458,180963130~8162978,189795676~16239675,207668912~4004045,212065768~8558568,246836530~4250329,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~5127042,329343811~2701118,336800196~9967906,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.728 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78839889~9212761,92274688~12071468,104857600~4221929,109482434~10542390,121634816~13523600,143026722~4018474,147439418~4104761,178154435~2311458,180963130~8162978,189795676~16239675,207668912~12955424,245550311~5536548,254724310~2114882,260046848~5747696,274173672~5897142,293991835~5220209,306184192~11718094,322513851~5127042,329343811~2701118,335544320~11223782,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.729 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 17 diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57640836~6971573,65183907~3342866,72073222~2361176,78839889~9212761,88080139~16266017,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,254724310~11070234,269267167~10803647,295753178~3458866,307919109~9983177,318129639~337373,322513851~5127042,329343811~2701118,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.729 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~12253486,53365114~662488,57640836~6971573,65183907~3342866,72073222~2361176,78839889~9212761,88080139~16266017,106697534~2381995,109482434~10542390,124550033~2194459,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,254724310~11070234,269267167~10803647,295753178~3458866,307919109~9983177,318129639~337373,322513851~5127042,329343811~2701118,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.734 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78839889~9212761,88080139~16266017,104857600~4221929,109482434~10542390,121634816~13523600,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,245550311~5536548,254724310~11070234,268435456~11635358,293991835~5220209,306184192~11718094,318129639~337373,322513851~5127042,329343811~2701118,335544320~11223782,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.734 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 18 diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,254724310~11070234,269267167~10803647,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~5127042,329343811~2701118,333824662~117846,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.734 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,254724310~11070234,269267167~10803647,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~5127042,329343811~2701118,333824662~117846,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.740 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,245550311~5536548,254724310~11070234,268435456~11635358,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~5127042,329343811~4598697,335544320~11223782,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.740 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 19 diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.740 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.746 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:48.746 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 9 diff [39741677~6065998,47096226~1496609,143026722~1320660,192736931~5856241,207668912~4004045,263765730~2028814,382357326~6599012,397672230~6203578] 2024-03-29T05:08:48.746 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [39741677~6065998,47096226~1496609,143026722~1320660,192736931~5856241,207668912~4004045,263765730~2028814,382357326~6599012,397672230~6203578] 2024-03-29T05:08:48.751 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [39741677~6065998,47096226~1496609,143026722~1320660,192736931~5856241,207668912~4004045,260046848~5747696,382357326~6599012,394264576~9611232] 2024-03-29T05:08:48.751 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 10 diff [34926731~2450064,39741677~10242690,72073222~2361176,81352804~3099543,111888563~6642772,143026722~1320660,192736931~5856241,200988779~4276349,207668912~4004045,263765730~2028814,295753178~3458866,322680407~1513294,369311906~8107396,382357326~6599012,397672230~6203578,407807916~5424201] 2024-03-29T05:08:48.751 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [34926731~2450064,39741677~10242690,72073222~2361176,81352804~3099543,111888563~6642772,143026722~1320660,192736931~5856241,200988779~4276349,207668912~4004045,263765730~2028814,295753178~3458866,322680407~1513294,369311906~8107396,382357326~6599012,397672230~6203578,407807916~5424201] 2024-03-29T05:08:48.757 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [34926731~2450064,39741677~10242690,72073222~2361176,81352804~3099543,111888563~6642772,143026722~1320660,192736931~5856241,200988779~4276349,207668912~4004045,260046848~5747696,293991835~5220209,322680407~1513294,369311906~8107396,382357326~6599012,394264576~9611232,407807916~5424201] 2024-03-29T05:08:48.757 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 11 diff [34926731~2450064,39741677~10242690,59139709~4018447,72073222~2361176,81352804~3099543,111888563~6642772,143026722~1320660,147439418~4104761,189795676~8797496,200988779~4276349,207668912~4004045,213342593~7281743,263765730~2028814,295753178~3458866,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~12423231,397672230~6203578,407807916~5424201] 2024-03-29T05:08:48.757 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [34926731~2450064,39741677~10242690,59139709~4018447,72073222~2361176,81352804~3099543,111888563~6642772,143026722~1320660,147439418~4104761,189795676~8797496,200988779~4276349,207668912~4004045,213342593~7281743,263765730~2028814,295753178~3458866,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~12423231,397672230~6203578,407807916~5424201] 2024-03-29T05:08:48.762 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [34926731~2450064,39741677~10242690,59139709~4018447,72073222~2361176,81352804~3099543,111888563~6642772,143026722~1320660,147439418~4104761,189795676~8797496,200988779~4276349,207668912~12955424,260046848~5747696,293991835~5220209,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~21518482,407807916~5424201] 2024-03-29T05:08:48.762 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 12 diff [28709632~1902975,34926731~2450064,39741677~10242690,59139709~4018447,72073222~2361176,78839889~5612458,111888563~6642772,143026722~1320660,147439418~4104761,183855054~107719,189795676~10490677,200988779~4276349,207668912~4004045,213342593~7281743,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~12423231,397672230~6203578,407807916~5424201] 2024-03-29T05:08:48.762 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [28709632~1902975,34926731~2450064,39741677~10242690,59139709~4018447,72073222~2361176,78839889~5612458,111888563~6642772,143026722~1320660,147439418~4104761,183855054~107719,189795676~10490677,200988779~4276349,207668912~4004045,213342593~7281743,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~12423231,397672230~6203578,407807916~5424201] 2024-03-29T05:08:48.768 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~5446783,34926731~2450064,39741677~10242690,59139709~4018447,72073222~2361176,78839889~5612458,111888563~6642772,143026722~1320660,147439418~4104761,183855054~107719,189795676~10490677,200988779~4276349,207668912~12955424,254724310~2114882,260046848~5747696,274173672~5897142,293991835~5220209,306184192~11718094,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,382357326~21518482,407807916~5424201] 2024-03-29T05:08:48.768 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 13 diff [18311766~6078807,28709632~1902975,34926731~2450064,39741677~10242690,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,109482434~9048901,127176832~7981584,143026722~1320660,147439418~4104761,183855054~5271054,189795676~10490677,200988779~4276349,207668912~4004045,213342593~7281743,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,365857476~11561826,382357326~21518482,407807916~5424201] 2024-03-29T05:08:48.768 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [18311766~6078807,28709632~1902975,34926731~2450064,39741677~10242690,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,109482434~9048901,127176832~7981584,143026722~1320660,147439418~4104761,183855054~5271054,189795676~10490677,200988779~4276349,207668912~4004045,213342593~7281743,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,365857476~11561826,382357326~21518482,407807916~5424201] 2024-03-29T05:08:48.773 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [18311766~6078807,25165824~5446783,34926731~2450064,39741677~10242690,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,109482434~9048901,126744492~8413924,143026722~1320660,147439418~4104761,183855054~5271054,189795676~10490677,200988779~4276349,207668912~12955424,254724310~2114882,260046848~5747696,274173672~5897142,293991835~5220209,306184192~11718094,322513851~1989805,329343811~2701118,335544320~11223782,365857476~11561826,382357326~21518482,407807916~5424201] 2024-03-29T05:08:48.774 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 14 diff [18311766~6078807,27077192~6988326,34926731~2450064,39741677~10242690,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,93785211~7481883,109482434~9048901,127176832~7981584,143026722~4018474,147439418~4104761,183855054~5271054,189795676~15469452,207668912~4004045,213342593~7281743,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,382357326~24009307,407807916~5424201] 2024-03-29T05:08:48.774 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [18311766~6078807,27077192~6988326,34926731~2450064,39741677~10242690,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,93785211~7481883,109482434~9048901,127176832~7981584,143026722~4018474,147439418~4104761,183855054~5271054,189795676~15469452,207668912~4004045,213342593~7281743,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,382357326~24009307,407807916~5424201] 2024-03-29T05:08:48.779 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [18311766~6078807,25165824~8899694,34926731~2450064,39741677~10242690,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,92274688~8992406,109482434~9048901,126744492~8413924,143026722~4018474,147439418~4104761,183855054~5271054,189795676~15469452,207668912~12955424,254724310~2114882,260046848~5747696,274173672~5897142,293991835~5220209,306184192~11718094,322513851~1989805,329343811~2701118,335544320~11223782,349983031~6849925,365857476~12524908,382357326~24009307,407807916~5424201] 2024-03-29T05:08:48.779 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 15 diff [607349~4280051,18311766~6078807,27077192~10396580,39741677~10242690,50354796~1640367,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,84964601~734981,93785211~10560945,109482434~9048901,127176832~7981584,143026722~4018474,147439418~4104761,183855054~5271054,189795676~16239675,207668912~4004045,212065768~8558568,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,381994856~24371777,407807916~5424201] 2024-03-29T05:08:48.779 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~4280051,18311766~6078807,27077192~10396580,39741677~10242690,50354796~1640367,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,84964601~734981,93785211~10560945,109482434~9048901,127176832~7981584,143026722~4018474,147439418~4104761,183855054~5271054,189795676~16239675,207668912~4004045,212065768~8558568,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,381994856~24371777,407807916~5424201] 2024-03-29T05:08:48.785 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~4280051,18311766~6078807,25165824~12307948,39741677~10242690,50354796~1640367,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,84964601~734981,92274688~12071468,109482434~9048901,126744492~8413924,143026722~4018474,147439418~4104761,183855054~5271054,189795676~16239675,207668912~12955424,254724310~2114882,260046848~5747696,274173672~5897142,293991835~5220209,306184192~11718094,322513851~1989805,329343811~2701118,335544320~11223782,349983031~6849925,365857476~12524908,381994856~24371777,407807916~5424201] 2024-03-29T05:08:48.785 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 16 diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~10242690,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,72073222~2361176,78839889~9212761,93785211~10560945,108145176~934353,109482434~10542390,127176832~7981584,143026722~4018474,147439418~4104761,183855054~5271054,189795676~16239675,207668912~4004045,212065768~8558568,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~5127042,329343811~2701118,336800196~9967906,349983031~6849925,357994231~2693797,365857476~12524908,381700185~24666448,407807916~5424201] 2024-03-29T05:08:48.785 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~10242690,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,72073222~2361176,78839889~9212761,93785211~10560945,108145176~934353,109482434~10542390,127176832~7981584,143026722~4018474,147439418~4104761,183855054~5271054,189795676~16239675,207668912~4004045,212065768~8558568,254724310~2114882,263765730~2028814,274173672~5897142,295753178~3458866,307919109~9983177,322513851~5127042,329343811~2701118,336800196~9967906,349983031~6849925,357994231~2693797,365857476~12524908,381700185~24666448,407807916~5424201] 2024-03-29T05:08:48.791 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,39741677~10242690,50354796~3672806,57640836~6971573,66741740~1785033,72073222~2361176,78839889~9212761,92274688~12071468,108145176~934353,109482434~10542390,126744492~8413924,143026722~4018474,147439418~4104761,183855054~5271054,189795676~16239675,207668912~12955424,254724310~2114882,260046848~5747696,274173672~5897142,293991835~5220209,306184192~11718094,322513851~5127042,329343811~2701118,335544320~11223782,349983031~6849925,357994231~2693797,365857476~12524908,381700185~24666448,407807916~5424201] 2024-03-29T05:08:48.791 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 17 diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~10242690,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,72073222~2361176,78839889~9212761,88080139~16266017,108145176~934353,109482434~10542390,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,254724310~11070234,269267167~10803647,295753178~3458866,307919109~9983177,318129639~337373,322513851~5127042,329343811~2701118,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~24666448,407807916~5424201] 2024-03-29T05:08:48.791 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~10242690,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,72073222~2361176,78839889~9212761,88080139~16266017,108145176~934353,109482434~10542390,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,254724310~11070234,269267167~10803647,295753178~3458866,307919109~9983177,318129639~337373,322513851~5127042,329343811~2701118,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~24666448,407807916~5424201] 2024-03-29T05:08:48.796 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,39741677~10242690,50354796~3672806,57640836~6971573,66741740~1785033,72073222~2361176,78839889~9212761,88080139~16266017,108145176~934353,109482434~10542390,126744492~8413924,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,254724310~11070234,268435456~11635358,293991835~5220209,306184192~11718094,318129639~337373,322513851~5127042,329343811~2701118,335544320~11223782,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~24666448,407807916~5424201] 2024-03-29T05:08:48.796 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 18 diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,254724310~11070234,269267167~10803647,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~5127042,329343811~2701118,333824662~117846,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:48.796 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,254724310~11070234,269267167~10803647,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~5127042,329343811~2701118,333824662~117846,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:48.802 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,254724310~11070234,268435456~11635358,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~5127042,329343811~4598697,335544320~11223782,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:48.802 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 19 diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:48.802 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:48.807 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:48.808 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 10 diff [34926731~2450064,45179185~4805182,72073222~2361176,81352804~3099543,111888563~6642772,200988779~4276349,295753178~3458866,322680407~1513294,369311906~8107396,407807916~5424201] 2024-03-29T05:08:48.808 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [34926731~2450064,45179185~4805182,72073222~2361176,81352804~3099543,111888563~6642772,200988779~4276349,295753178~3458866,322680407~1513294,369311906~8107396,407807916~5424201] 2024-03-29T05:08:48.813 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [34926731~2450064,45179185~4805182,72073222~2361176,81352804~3099543,111888563~6642772,200988779~4276349,293991835~5220209,322680407~1513294,369311906~8107396,407807916~5424201] 2024-03-29T05:08:48.813 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 11 diff [34926731~2450064,45179185~4805182,59139709~4018447,72073222~2361176,81352804~3099543,111888563~6642772,147439418~4104761,189795676~6858786,200988779~4276349,213342593~7281743,295753178~3458866,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,383035237~1558455,387938464~6842093,407807916~5424201] 2024-03-29T05:08:48.813 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [34926731~2450064,45179185~4805182,59139709~4018447,72073222~2361176,81352804~3099543,111888563~6642772,147439418~4104761,189795676~6858786,200988779~4276349,213342593~7281743,295753178~3458866,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,383035237~1558455,387938464~6842093,407807916~5424201] 2024-03-29T05:08:48.818 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [34926731~2450064,45179185~4805182,59139709~4018447,72073222~2361176,81352804~3099543,111888563~6642772,147439418~4104761,189795676~6858786,200988779~4276349,211672957~8951379,293991835~5220209,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,383035237~1558455,387938464~6842093,407807916~5424201] 2024-03-29T05:08:48.818 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 12 diff [28709632~1902975,34926731~2450064,45179185~4805182,59139709~4018447,72073222~2361176,78839889~5612458,111888563~6642772,147439418~4104761,183855054~107719,189795676~10490677,200988779~4276349,213342593~7281743,254724310~2114882,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,383035237~1558455,387938464~6842093,407807916~5424201] 2024-03-29T05:08:48.819 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [28709632~1902975,34926731~2450064,45179185~4805182,59139709~4018447,72073222~2361176,78839889~5612458,111888563~6642772,147439418~4104761,183855054~107719,189795676~10490677,200988779~4276349,213342593~7281743,254724310~2114882,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,383035237~1558455,387938464~6842093,407807916~5424201] 2024-03-29T05:08:48.824 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~5446783,34926731~2450064,45179185~4805182,59139709~4018447,72073222~2361176,78839889~5612458,111888563~6642772,147439418~4104761,183855054~107719,189795676~10490677,200988779~4276349,211672957~8951379,254724310~2114882,274173672~5897142,293991835~5220209,306184192~11718094,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,369311906~8107396,383035237~1558455,387938464~6842093,407807916~5424201] 2024-03-29T05:08:48.824 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 13 diff [18311766~6078807,28709632~1902975,34926731~2450064,45179185~4805182,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,109482434~9048901,127176832~7981584,147439418~4104761,183855054~5271054,189795676~10490677,200988779~4276349,213342593~7281743,254724310~2114882,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,365857476~11561826,383035237~1558455,387938464~10121736,407807916~5424201] 2024-03-29T05:08:48.824 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [18311766~6078807,28709632~1902975,34926731~2450064,45179185~4805182,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,109482434~9048901,127176832~7981584,147439418~4104761,183855054~5271054,189795676~10490677,200988779~4276349,213342593~7281743,254724310~2114882,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,365857476~11561826,383035237~1558455,387938464~10121736,407807916~5424201] 2024-03-29T05:08:48.830 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [18311766~6078807,25165824~5446783,34926731~2450064,45179185~4805182,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,109482434~9048901,126744492~8413924,147439418~4104761,183855054~5271054,189795676~10490677,200988779~4276349,211672957~8951379,254724310~2114882,274173672~5897142,293991835~5220209,306184192~11718094,322513851~1989805,329343811~2701118,335544320~11223782,365857476~11561826,383035237~1558455,387938464~10121736,407807916~5424201] 2024-03-29T05:08:48.830 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 14 diff [18311766~6078807,27077192~6988326,34926731~2450064,45179185~4805182,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,93785211~7481883,109482434~9048901,127176832~7981584,143272389~3772807,147439418~4104761,183855054~5271054,189795676~15469452,213342593~7281743,254724310~2114882,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,383035237~1558455,387938464~10121736,399456201~6910432,407807916~5424201] 2024-03-29T05:08:48.830 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [18311766~6078807,27077192~6988326,34926731~2450064,45179185~4805182,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,93785211~7481883,109482434~9048901,127176832~7981584,143272389~3772807,147439418~4104761,183855054~5271054,189795676~15469452,213342593~7281743,254724310~2114882,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,383035237~1558455,387938464~10121736,399456201~6910432,407807916~5424201] 2024-03-29T05:08:48.835 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [18311766~6078807,25165824~8899694,34926731~2450064,45179185~4805182,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,92274688~8992406,109482434~9048901,126744492~8413924,143272389~3772807,147439418~4104761,183855054~5271054,189795676~15469452,211672957~8951379,254724310~2114882,274173672~5897142,293991835~5220209,306184192~11718094,322513851~1989805,329343811~2701118,335544320~11223782,349983031~6849925,365857476~12524908,383035237~1558455,387938464~10121736,399456201~6910432,407807916~5424201] 2024-03-29T05:08:48.835 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 15 diff [607349~4280051,18311766~6078807,27077192~10396580,45179185~4805182,50354796~1640367,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,84964601~734981,93785211~10560945,109482434~9048901,127176832~7981584,143272389~3772807,147439418~4104761,183855054~5271054,189795676~16239675,212065768~8558568,254724310~2114882,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,381994856~16065344,399456201~6910432,407807916~5424201] 2024-03-29T05:08:48.836 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~4280051,18311766~6078807,27077192~10396580,45179185~4805182,50354796~1640367,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,84964601~734981,93785211~10560945,109482434~9048901,127176832~7981584,143272389~3772807,147439418~4104761,183855054~5271054,189795676~16239675,212065768~8558568,254724310~2114882,274173672~5897142,295753178~3458866,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~12524908,381994856~16065344,399456201~6910432,407807916~5424201] 2024-03-29T05:08:48.841 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~4280051,18311766~6078807,25165824~12307948,45179185~4805182,50354796~1640367,57640836~6971573,66741740~1785033,72073222~2361176,78839889~5612458,84964601~734981,92274688~12071468,109482434~9048901,126744492~8413924,143272389~3772807,147439418~4104761,183855054~5271054,189795676~16239675,211672957~8951379,254724310~2114882,274173672~5897142,293991835~5220209,306184192~11718094,322513851~1989805,329343811~2701118,335544320~11223782,349983031~6849925,365857476~12524908,381994856~16065344,399456201~6910432,407807916~5424201] 2024-03-29T05:08:48.841 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 16 diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~4805182,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,72073222~2361176,78839889~9212761,93785211~10560945,108145176~934353,109482434~10542390,127176832~7981584,143272389~3772807,147439418~4104761,183855054~5271054,189795676~16239675,212065768~8558568,254724310~2114882,274173672~5897142,295753178~3458866,307919109~9983177,322513851~5127042,329343811~2701118,336800196~9967906,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,399456201~6910432,407807916~5424201] 2024-03-29T05:08:48.841 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~4805182,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,72073222~2361176,78839889~9212761,93785211~10560945,108145176~934353,109482434~10542390,127176832~7981584,143272389~3772807,147439418~4104761,183855054~5271054,189795676~16239675,212065768~8558568,254724310~2114882,274173672~5897142,295753178~3458866,307919109~9983177,322513851~5127042,329343811~2701118,336800196~9967906,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,399456201~6910432,407807916~5424201] 2024-03-29T05:08:48.846 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,45179185~4805182,50354796~3672806,57640836~6971573,66741740~1785033,72073222~2361176,78839889~9212761,92274688~12071468,108145176~934353,109482434~10542390,126744492~8413924,143272389~3772807,147439418~4104761,183855054~5271054,189795676~16239675,211672957~8951379,254724310~2114882,274173672~5897142,293991835~5220209,306184192~11718094,322513851~5127042,329343811~2701118,335544320~11223782,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,399456201~6910432,407807916~5424201] 2024-03-29T05:08:48.846 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 17 diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~4805182,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,72073222~2361176,78839889~9212761,88080139~16266017,108145176~934353,109482434~10542390,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,254724310~9080851,269267167~10803647,295753178~3458866,307919109~9983177,318129639~337373,322513851~5127042,329343811~2701118,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,399456201~6910432,407807916~5424201] 2024-03-29T05:08:48.846 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~4805182,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,72073222~2361176,78839889~9212761,88080139~16266017,108145176~934353,109482434~10542390,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,254724310~9080851,269267167~10803647,295753178~3458866,307919109~9983177,318129639~337373,322513851~5127042,329343811~2701118,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,399456201~6910432,407807916~5424201] 2024-03-29T05:08:48.852 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,45179185~4805182,50354796~3672806,57640836~6971573,66741740~1785033,72073222~2361176,78839889~9212761,88080139~16266017,108145176~934353,109482434~10542390,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,254724310~9080851,268435456~11635358,293991835~5220209,306184192~11718094,318129639~337373,322513851~5127042,329343811~2701118,335544320~11223782,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,399456201~6910432,407807916~5424201] 2024-03-29T05:08:48.852 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 18 diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,254724310~9080851,269267167~10803647,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~5127042,329343811~2701118,333824662~117846,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,399456201~7300153,407807916~5424201] 2024-03-29T05:08:48.852 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,254724310~9080851,269267167~10803647,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~5127042,329343811~2701118,333824662~117846,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,399456201~7300153,407807916~5424201] 2024-03-29T05:08:48.858 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,254724310~9080851,268435456~11635358,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~5127042,329343811~4598697,335544320~11223782,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,399456201~7300153,407807916~5424201] 2024-03-29T05:08:48.858 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 19 diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:48.858 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:48.863 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:48.863 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 11 diff [59139709~4018447,147439418~4104761,189795676~6858786,213342593~7281743,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,383035237~1558455,387938464~6842093] 2024-03-29T05:08:48.864 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [59139709~4018447,147439418~4104761,189795676~6858786,213342593~7281743,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,383035237~1558455,387938464~6842093] 2024-03-29T05:08:48.869 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [59139709~4018447,147439418~4104761,189795676~6858786,211672957~8951379,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,383035237~1558455,387938464~6842093] 2024-03-29T05:08:48.869 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 12 diff [28709632~1902975,59139709~4018447,78839889~3325874,147439418~4104761,183855054~107719,189795676~10490677,213342593~7281743,254724310~2114882,274173672~5897142,307919109~9983177,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,383035237~1558455,387938464~6842093] 2024-03-29T05:08:48.869 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [28709632~1902975,59139709~4018447,78839889~3325874,147439418~4104761,183855054~107719,189795676~10490677,213342593~7281743,254724310~2114882,274173672~5897142,307919109~9983177,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,383035237~1558455,387938464~6842093] 2024-03-29T05:08:48.874 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~5446783,59139709~4018447,78839889~3325874,147439418~4104761,183855054~107719,189795676~10490677,211672957~8951379,254724310~2114882,274173672~5897142,306184192~11718094,322513851~1989805,329343811~2701118,340147706~6620396,366005835~1027758,383035237~1558455,387938464~6842093] 2024-03-29T05:08:48.874 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 13 diff [18311766~6078807,28709632~1902975,57640836~6971573,66741740~1785033,78839889~3325874,109482434~4861457,127176832~7981584,147439418~4104761,183855054~5271054,189795676~10490677,213342593~7281743,254724310~2114882,274173672~5897142,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,365857476~4405229,383035237~1558455,387938464~10121736] 2024-03-29T05:08:48.874 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [18311766~6078807,28709632~1902975,57640836~6971573,66741740~1785033,78839889~3325874,109482434~4861457,127176832~7981584,147439418~4104761,183855054~5271054,189795676~10490677,213342593~7281743,254724310~2114882,274173672~5897142,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,365857476~4405229,383035237~1558455,387938464~10121736] 2024-03-29T05:08:48.880 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [18311766~6078807,25165824~5446783,57640836~6971573,66741740~1785033,78839889~3325874,109482434~4861457,126744492~8413924,147439418~4104761,183855054~5271054,189795676~10490677,211672957~8951379,254724310~2114882,274173672~5897142,306184192~11718094,322513851~1989805,329343811~2701118,335544320~11223782,365857476~4405229,383035237~1558455,387938464~10121736] 2024-03-29T05:08:48.880 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 14 diff [18311766~6078807,27077192~6988326,57640836~6971573,66741740~1785033,78839889~3325874,93785211~7481883,109482434~4861457,127176832~7981584,143272389~3772807,147439418~4104761,183855054~5271054,189795676~12547062,213342593~7281743,254724310~2114882,274173672~5897142,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~4405229,375001040~3381344,383035237~1558455,387938464~10121736,399456201~6910432] 2024-03-29T05:08:48.880 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [18311766~6078807,27077192~6988326,57640836~6971573,66741740~1785033,78839889~3325874,93785211~7481883,109482434~4861457,127176832~7981584,143272389~3772807,147439418~4104761,183855054~5271054,189795676~12547062,213342593~7281743,254724310~2114882,274173672~5897142,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~4405229,375001040~3381344,383035237~1558455,387938464~10121736,399456201~6910432] 2024-03-29T05:08:48.886 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [18311766~6078807,25165824~8899694,57640836~6971573,66741740~1785033,78839889~3325874,92274688~8992406,109482434~4861457,126744492~8413924,143272389~3772807,147439418~4104761,183855054~5271054,189795676~12547062,211672957~8951379,254724310~2114882,274173672~5897142,306184192~11718094,322513851~1989805,329343811~2701118,335544320~11223782,349983031~6849925,365857476~4405229,375001040~3381344,383035237~1558455,387938464~10121736,399456201~6910432] 2024-03-29T05:08:48.886 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 15 diff [607349~4280051,18311766~6078807,27077192~10396580,50354796~1640367,57640836~6971573,66741740~1785033,78839889~3325874,84964601~734981,93785211~10560945,109482434~4861457,127176832~7981584,143272389~3772807,147439418~4104761,183855054~5271054,189795676~12547062,202433993~3601358,212065768~8558568,254724310~2114882,274173672~5897142,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~4405229,375001040~3381344,381994856~16065344,399456201~6910432] 2024-03-29T05:08:48.886 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~4280051,18311766~6078807,27077192~10396580,50354796~1640367,57640836~6971573,66741740~1785033,78839889~3325874,84964601~734981,93785211~10560945,109482434~4861457,127176832~7981584,143272389~3772807,147439418~4104761,183855054~5271054,189795676~12547062,202433993~3601358,212065768~8558568,254724310~2114882,274173672~5897142,307919109~9983177,322513851~1989805,329343811~2701118,336800196~9967906,349983031~6849925,365857476~4405229,375001040~3381344,381994856~16065344,399456201~6910432] 2024-03-29T05:08:48.891 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~4280051,18311766~6078807,25165824~12307948,50354796~1640367,57640836~6971573,66741740~1785033,78839889~3325874,84964601~734981,92274688~12071468,109482434~4861457,126744492~8413924,143272389~3772807,147439418~4104761,183855054~5271054,189795676~12547062,202433993~3601358,211672957~8951379,254724310~2114882,274173672~5897142,306184192~11718094,322513851~1989805,329343811~2701118,335544320~11223782,349983031~6849925,365857476~4405229,375001040~3381344,381994856~16065344,399456201~6910432] 2024-03-29T05:08:48.891 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 16 diff [607349~5934701,18311766~6078807,27077192~10396580,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,78839889~9212761,93785211~10560945,108145176~934353,109482434~4861457,114910182~5114642,127176832~7981584,143272389~3772807,147439418~4104761,183855054~5271054,189795676~12547062,202433993~3601358,212065768~8558568,254724310~2114882,274173672~5897142,307919109~9983177,322513851~5127042,329343811~2701118,336800196~9967906,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~16360015,399456201~6910432] 2024-03-29T05:08:48.891 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,78839889~9212761,93785211~10560945,108145176~934353,109482434~4861457,114910182~5114642,127176832~7981584,143272389~3772807,147439418~4104761,183855054~5271054,189795676~12547062,202433993~3601358,212065768~8558568,254724310~2114882,274173672~5897142,307919109~9983177,322513851~5127042,329343811~2701118,336800196~9967906,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~16360015,399456201~6910432] 2024-03-29T05:08:48.897 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,50354796~3672806,57640836~6971573,66741740~1785033,78839889~9212761,92274688~12071468,108145176~934353,109482434~4861457,114910182~5114642,126744492~8413924,143272389~3772807,147439418~4104761,183855054~5271054,189795676~12547062,202433993~3601358,211672957~8951379,254724310~2114882,274173672~5897142,306184192~11718094,322513851~5127042,329343811~2701118,335544320~11223782,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~16360015,399456201~6910432] 2024-03-29T05:08:48.897 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 17 diff [607349~5934701,18311766~6078807,27077192~10396580,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,78839889~9212761,88080139~16266017,108145176~934353,109482434~4861457,114910182~5114642,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~12547062,202433993~3601358,208104136~550405,211369000~9255336,254724310~9080851,269267167~10803647,307919109~9983177,318129639~337373,322513851~5127042,329343811~2701118,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~16360015,399456201~6910432] 2024-03-29T05:08:48.897 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,78839889~9212761,88080139~16266017,108145176~934353,109482434~4861457,114910182~5114642,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~12547062,202433993~3601358,208104136~550405,211369000~9255336,254724310~9080851,269267167~10803647,307919109~9983177,318129639~337373,322513851~5127042,329343811~2701118,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~16360015,399456201~6910432] 2024-03-29T05:08:48.903 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,50354796~3672806,57640836~6971573,66741740~1785033,78839889~9212761,88080139~16266017,108145176~934353,109482434~4861457,114910182~5114642,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~12547062,202433993~3601358,208104136~550405,211369000~9255336,254724310~9080851,268435456~11635358,306184192~11718094,318129639~337373,322513851~5127042,329343811~2701118,335544320~11223782,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~16360015,399456201~6910432] 2024-03-29T05:08:48.903 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 18 diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,254724310~9080851,269267167~10803647,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~5127042,329343811~2701118,333824662~117846,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~16360015,399456201~7300153] 2024-03-29T05:08:48.903 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,254724310~9080851,269267167~10803647,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~5127042,329343811~2701118,333824662~117846,336122526~10645576,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~16360015,399456201~7300153] 2024-03-29T05:08:48.908 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,254724310~9080851,268435456~11635358,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,322513851~5127042,329343811~4598697,335544320~11223782,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~16360015,399456201~7300153] 2024-03-29T05:08:48.908 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 19 diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:48.908 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:48.914 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:48.914 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to 12 diff [28709632~1902975,78839889~3325874,183855054~107719,195956946~4329407,254724310~2114882,274173672~5897142,307919109~9983177] 2024-03-29T05:08:48.914 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [28709632~1902975,78839889~3325874,183855054~107719,195956946~4329407,254724310~2114882,274173672~5897142,307919109~9983177] 2024-03-29T05:08:48.919 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~5446783,78839889~3325874,183855054~107719,195956946~4329407,254724310~2114882,274173672~5897142,306184192~11718094] 2024-03-29T05:08:48.919 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to 13 diff [18311766~6078807,28709632~1902975,57640836~6971573,66741740~1785033,78839889~3325874,109482434~4861457,127176832~7981584,183855054~5271054,195956946~4329407,254724310~2114882,274173672~5897142,307919109~9983177,336800196~6585868,365857476~4405229,394051856~4008344] 2024-03-29T05:08:48.919 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [18311766~6078807,28709632~1902975,57640836~6971573,66741740~1785033,78839889~3325874,109482434~4861457,127176832~7981584,183855054~5271054,195956946~4329407,254724310~2114882,274173672~5897142,307919109~9983177,336800196~6585868,365857476~4405229,394051856~4008344] 2024-03-29T05:08:48.924 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [18311766~6078807,25165824~5446783,57640836~6971573,66741740~1785033,78839889~3325874,109482434~4861457,126744492~8413924,183855054~5271054,195956946~4329407,254724310~2114882,274173672~5897142,306184192~11718094,335544320~7841744,365857476~4405229,394051856~4008344] 2024-03-29T05:08:48.924 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to 14 diff [18311766~6078807,27077192~6988326,57640836~6971573,66741740~1785033,78839889~3325874,93785211~7481883,109482434~4861457,127176832~7981584,143272389~3772807,183855054~5271054,195956946~6385792,254724310~2114882,274173672~5897142,307919109~9983177,336800196~6585868,349983031~6849925,365857476~4405229,375001040~3381344,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.925 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [18311766~6078807,27077192~6988326,57640836~6971573,66741740~1785033,78839889~3325874,93785211~7481883,109482434~4861457,127176832~7981584,143272389~3772807,183855054~5271054,195956946~6385792,254724310~2114882,274173672~5897142,307919109~9983177,336800196~6585868,349983031~6849925,365857476~4405229,375001040~3381344,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.930 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [18311766~6078807,25165824~8899694,57640836~6971573,66741740~1785033,78839889~3325874,92274688~8992406,109482434~4861457,126744492~8413924,143272389~3772807,183855054~5271054,195956946~6385792,254724310~2114882,274173672~5897142,306184192~11718094,335544320~7841744,349983031~6849925,365857476~4405229,375001040~3381344,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.930 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to 15 diff [607349~4280051,18311766~6078807,27077192~10396580,50354796~1640367,57640836~6971573,66741740~1785033,78839889~3325874,84964601~734981,93785211~10560945,109482434~4861457,127176832~7981584,143272389~3772807,183855054~5271054,195956946~6385792,202433993~3601358,212065768~7948638,254724310~2114882,274173672~5897142,307919109~9983177,336800196~6585868,349983031~6849925,365857476~4405229,375001040~3381344,381994856~6144098,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.930 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~4280051,18311766~6078807,27077192~10396580,50354796~1640367,57640836~6971573,66741740~1785033,78839889~3325874,84964601~734981,93785211~10560945,109482434~4861457,127176832~7981584,143272389~3772807,183855054~5271054,195956946~6385792,202433993~3601358,212065768~7948638,254724310~2114882,274173672~5897142,307919109~9983177,336800196~6585868,349983031~6849925,365857476~4405229,375001040~3381344,381994856~6144098,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.936 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~4280051,18311766~6078807,25165824~12307948,50354796~1640367,57640836~6971573,66741740~1785033,78839889~3325874,84964601~734981,92274688~12071468,109482434~4861457,126744492~8413924,143272389~3772807,183855054~5271054,195956946~6385792,202433993~3601358,212065768~7948638,254724310~2114882,274173672~5897142,306184192~11718094,335544320~7841744,349983031~6849925,365857476~4405229,375001040~3381344,381994856~6144098,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.936 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to 16 diff [607349~5934701,18311766~6078807,27077192~10396580,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,78839889~9212761,93785211~10560945,108145176~934353,109482434~4861457,114910182~5114642,127176832~7981584,143272389~3772807,183855054~5271054,195956946~6385792,202433993~3601358,212065768~7948638,254724310~2114882,274173672~5897142,307919109~9983177,323647203~3993690,336800196~6585868,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.936 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,78839889~9212761,93785211~10560945,108145176~934353,109482434~4861457,114910182~5114642,127176832~7981584,143272389~3772807,183855054~5271054,195956946~6385792,202433993~3601358,212065768~7948638,254724310~2114882,274173672~5897142,307919109~9983177,323647203~3993690,336800196~6585868,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.941 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,50354796~3672806,57640836~6971573,66741740~1785033,78839889~9212761,92274688~12071468,108145176~934353,109482434~4861457,114910182~5114642,126744492~8413924,143272389~3772807,183855054~5271054,195956946~6385792,202433993~3601358,212065768~7948638,254724310~2114882,274173672~5897142,306184192~11718094,323647203~3993690,335544320~7841744,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.941 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to 17 diff [607349~5934701,18311766~6078807,27077192~10396580,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,78839889~9212761,88080139~16266017,108145176~934353,109482434~4861457,114910182~5114642,127176832~7981584,143272389~3772807,176445827~12680281,195956946~6385792,202433993~3601358,208104136~550405,211369000~8645406,254724310~9080851,269267167~10803647,307919109~9983177,318129639~337373,323647203~3993690,336122526~7263538,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.941 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,78839889~9212761,88080139~16266017,108145176~934353,109482434~4861457,114910182~5114642,127176832~7981584,143272389~3772807,176445827~12680281,195956946~6385792,202433993~3601358,208104136~550405,211369000~8645406,254724310~9080851,269267167~10803647,307919109~9983177,318129639~337373,323647203~3993690,336122526~7263538,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.947 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,50354796~3672806,57640836~6971573,66741740~1785033,78839889~9212761,88080139~16266017,108145176~934353,109482434~4861457,114910182~5114642,126744492~8413924,143272389~3772807,176445827~12680281,195956946~6385792,202433993~3601358,208104136~550405,211369000~8645406,254724310~9080851,268435456~11635358,306184192~11718094,318129639~337373,323647203~3993690,335544320~7841744,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.947 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to 18 diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,254724310~9080851,269267167~10803647,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~3993690,333824662~117846,336122526~7263538,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~7300153] 2024-03-29T05:08:48.947 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,254724310~9080851,269267167~10803647,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~3993690,333824662~117846,336122526~7263538,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~7300153] 2024-03-29T05:08:48.952 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,254724310~9080851,268435456~11635358,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,323647203~3993690,332044929~1897579,335544320~7841744,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~7300153] 2024-03-29T05:08:48.953 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to 19 diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:48.953 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:48.958 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,323647203~6256678,332044929~11341135,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:48.958 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to 13 diff [18311766~6078807,57640836~6971573,66741740~1785033,109482434~4861457,127176832~7981584,183940397~5185711,336800196~6585868,365857476~4405229,394051856~4008344] 2024-03-29T05:08:48.958 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [18311766~6078807,57640836~6971573,66741740~1785033,109482434~4861457,127176832~7981584,183940397~5185711,336800196~6585868,365857476~4405229,394051856~4008344] 2024-03-29T05:08:48.964 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [18311766~6078807,57640836~6971573,66741740~1785033,109482434~4861457,126744492~8413924,183940397~5185711,335544320~7841744,365857476~4405229,394051856~4008344] 2024-03-29T05:08:48.964 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to 14 diff [18311766~6078807,27077192~6988326,57640836~6971573,66741740~1785033,93785211~7481883,109482434~4861457,127176832~7981584,143272389~3772807,183940397~5185711,198059281~4283457,336800196~6585868,349983031~6849925,365857476~4405229,375001040~3381344,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.964 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [18311766~6078807,27077192~6988326,57640836~6971573,66741740~1785033,93785211~7481883,109482434~4861457,127176832~7981584,143272389~3772807,183940397~5185711,198059281~4283457,336800196~6585868,349983031~6849925,365857476~4405229,375001040~3381344,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.970 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [18311766~6078807,27077192~6988326,57640836~6971573,66741740~1785033,92274688~8992406,109482434~4861457,126744492~8413924,143272389~3772807,183940397~5185711,198059281~4283457,335544320~7841744,349983031~6849925,365857476~4405229,375001040~3381344,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.970 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to 15 diff [607349~4280051,18311766~6078807,27077192~10396580,50354796~1640367,57640836~6971573,66741740~1785033,84964601~734981,93785211~10560945,109482434~4861457,127176832~7981584,143272389~3772807,183940397~5185711,198059281~4283457,202433993~3601358,212065768~7948638,313401620~414791,336800196~6585868,349983031~6849925,365857476~4405229,375001040~3381344,381994856~6144098,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.970 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~4280051,18311766~6078807,27077192~10396580,50354796~1640367,57640836~6971573,66741740~1785033,84964601~734981,93785211~10560945,109482434~4861457,127176832~7981584,143272389~3772807,183940397~5185711,198059281~4283457,202433993~3601358,212065768~7948638,313401620~414791,336800196~6585868,349983031~6849925,365857476~4405229,375001040~3381344,381994856~6144098,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.975 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~4280051,18311766~6078807,27077192~10396580,50354796~1640367,57640836~6971573,66741740~1785033,84964601~734981,92274688~12071468,109482434~4861457,126744492~8413924,143272389~3772807,183940397~5185711,198059281~4283457,202433993~3601358,212065768~7948638,313401620~414791,335544320~7841744,349983031~6849925,365857476~4405229,375001040~3381344,381994856~6144098,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.975 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to 16 diff [607349~5934701,18311766~6078807,27077192~10396580,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,81195022~6857628,93785211~10560945,108145176~934353,109482434~4861457,114910182~5114642,127176832~7981584,143272389~3772807,183940397~5185711,198059281~4283457,202433993~3601358,212065768~7948638,313401620~414791,323647203~3993690,336800196~6585868,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.975 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,81195022~6857628,93785211~10560945,108145176~934353,109482434~4861457,114910182~5114642,127176832~7981584,143272389~3772807,183940397~5185711,198059281~4283457,202433993~3601358,212065768~7948638,313401620~414791,323647203~3993690,336800196~6585868,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.981 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,27077192~10396580,50354796~3672806,57640836~6971573,66741740~1785033,81195022~6857628,92274688~12071468,108145176~934353,109482434~4861457,114910182~5114642,126744492~8413924,143272389~3772807,183940397~5185711,198059281~4283457,202433993~3601358,212065768~7948638,313401620~414791,323647203~3993690,335544320~7841744,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.981 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to 17 diff [607349~5934701,18311766~6078807,27077192~10396580,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,81195022~6857628,88080139~16266017,108145176~934353,109482434~4861457,114910182~5114642,127176832~7981584,143272389~3772807,176445827~12680281,198059281~4283457,202433993~3601358,208104136~550405,211369000~8645406,256353754~7451407,269267167~7063560,313401620~414791,318129639~337373,323647203~3993690,336122526~7263538,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.981 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,50354796~1640367,53365114~662488,57640836~6971573,66741740~1785033,81195022~6857628,88080139~16266017,108145176~934353,109482434~4861457,114910182~5114642,127176832~7981584,143272389~3772807,176445827~12680281,198059281~4283457,202433993~3601358,208104136~550405,211369000~8645406,256353754~7451407,269267167~7063560,313401620~414791,318129639~337373,323647203~3993690,336122526~7263538,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.986 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,27077192~10396580,50354796~3672806,57640836~6971573,66741740~1785033,81195022~6857628,88080139~16266017,108145176~934353,109482434~4861457,114910182~5114642,126744492~8413924,143272389~3772807,176445827~12680281,198059281~4283457,202433993~3601358,208104136~550405,211369000~8645406,256353754~7451407,268435456~7895271,313401620~414791,318129639~337373,323647203~3993690,335544320~7841744,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~6910432] 2024-03-29T05:08:48.987 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to 18 diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~2135316,81195022~23151134,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,256353754~7451407,269267167~7063560,298106400~3193945,304147414~1639746,313401620~414791,318129639~337373,323647203~3993690,333824662~117846,336122526~7263538,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~7300153] 2024-03-29T05:08:48.987 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~2135316,81195022~23151134,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,256353754~7451407,269267167~7063560,298106400~3193945,304147414~1639746,313401620~414791,318129639~337373,323647203~3993690,333824662~117846,336122526~7263538,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~7300153] 2024-03-29T05:08:48.992 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~2135316,81195022~23151134,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,256353754~7451407,268435456~7895271,298106400~3193945,301989888~3797272,313401620~414791,318129639~337373,323647203~3993690,332044929~1897579,335544320~7841744,347366098~139655,349983031~6849925,357994231~2693797,365857476~4405229,375001040~3381344,381700185~7648272,391533675~33935,394051856~4008344,399456201~7300153] 2024-03-29T05:08:48.992 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to 19 diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:48.992 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:48.997 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11341135,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:48.998 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to 14 diff [27077192~6988326,59746300~3583961,93785211~7481883,143272389~3772807,198059281~4283457,337214045~3795808,349983031~6849925,375001040~3381344,399456201~6910432] 2024-03-29T05:08:48.998 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [27077192~6988326,59746300~3583961,93785211~7481883,143272389~3772807,198059281~4283457,337214045~3795808,349983031~6849925,375001040~3381344,399456201~6910432] 2024-03-29T05:08:49.003 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [27077192~6988326,59746300~3583961,92274688~8992406,143272389~3772807,198059281~4283457,337214045~3795808,349983031~6849925,375001040~3381344,399456201~6910432] 2024-03-29T05:08:49.003 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to 15 diff [607349~4280051,27077192~10396580,50354796~1640367,59746300~3583961,84964601~734981,93785211~10560945,143272389~3772807,198059281~4283457,202433993~3601358,212065768~7948638,313401620~414791,337214045~3795808,349983031~6849925,375001040~3381344,381994856~6144098,399456201~6910432] 2024-03-29T05:08:49.004 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~4280051,27077192~10396580,50354796~1640367,59746300~3583961,84964601~734981,93785211~10560945,143272389~3772807,198059281~4283457,202433993~3601358,212065768~7948638,313401620~414791,337214045~3795808,349983031~6849925,375001040~3381344,381994856~6144098,399456201~6910432] 2024-03-29T05:08:49.009 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~4280051,27077192~10396580,50354796~1640367,59746300~3583961,84964601~734981,92274688~12071468,143272389~3772807,198059281~4283457,202433993~3601358,212065768~7948638,313401620~414791,337214045~3795808,349983031~6849925,375001040~3381344,381994856~6144098,399456201~6910432] 2024-03-29T05:08:49.009 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to 16 diff [607349~5934701,27077192~10396580,50354796~1640367,53365114~662488,59746300~3583961,81195022~6857628,93785211~10560945,108145176~934353,114910182~5114642,143272389~3772807,198059281~4283457,202433993~3601358,212065768~7948638,313401620~414791,323647203~3993690,337214045~3795808,349983031~6849925,357994231~2693797,375001040~3381344,381700185~7648272,391533675~33935,399456201~6910432] 2024-03-29T05:08:49.009 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,27077192~10396580,50354796~1640367,53365114~662488,59746300~3583961,81195022~6857628,93785211~10560945,108145176~934353,114910182~5114642,143272389~3772807,198059281~4283457,202433993~3601358,212065768~7948638,313401620~414791,323647203~3993690,337214045~3795808,349983031~6849925,357994231~2693797,375001040~3381344,381700185~7648272,391533675~33935,399456201~6910432] 2024-03-29T05:08:49.016 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,27077192~10396580,50354796~3672806,59746300~3583961,81195022~6857628,92274688~12071468,108145176~934353,114910182~5114642,143272389~3772807,198059281~4283457,202433993~3601358,212065768~7948638,313401620~414791,323647203~3993690,337214045~3795808,349983031~6849925,357994231~2693797,375001040~3381344,381700185~7648272,391533675~33935,399456201~6910432] 2024-03-29T05:08:49.016 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to 17 diff [607349~5934701,27077192~10396580,50354796~1640367,53365114~662488,59746300~3583961,81195022~6857628,88080139~16266017,108145176~934353,114910182~5114642,143272389~3772807,176445827~7860687,198059281~4283457,202433993~3601358,208104136~550405,211369000~8645406,256353754~7451407,269267167~7063560,313401620~414791,318129639~337373,323647203~3993690,336122526~7073563,347366098~139655,349983031~6849925,357994231~2693797,375001040~3381344,381700185~7648272,391533675~33935,399456201~6910432] 2024-03-29T05:08:49.016 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,27077192~10396580,50354796~1640367,53365114~662488,59746300~3583961,81195022~6857628,88080139~16266017,108145176~934353,114910182~5114642,143272389~3772807,176445827~7860687,198059281~4283457,202433993~3601358,208104136~550405,211369000~8645406,256353754~7451407,269267167~7063560,313401620~414791,318129639~337373,323647203~3993690,336122526~7073563,347366098~139655,349983031~6849925,357994231~2693797,375001040~3381344,381700185~7648272,391533675~33935,399456201~6910432] 2024-03-29T05:08:49.022 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,27077192~10396580,50354796~3672806,59746300~3583961,81195022~6857628,88080139~16266017,108145176~934353,114910182~5114642,143272389~3772807,176445827~7860687,198059281~4283457,202433993~3601358,208104136~550405,211369000~8645406,256353754~7451407,268435456~7895271,313401620~414791,318129639~337373,323647203~3993690,336122526~7073563,347366098~139655,349983031~6849925,357994231~2693797,375001040~3381344,381700185~7648272,391533675~33935,399456201~6910432] 2024-03-29T05:08:49.023 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to 18 diff [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~2135316,81195022~23151134,108145176~934353,114910182~5114642,120464135~4299057,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,256353754~7451407,269267167~7063560,298106400~3193945,304147414~1639746,313401620~414791,318129639~337373,323647203~3993690,333824662~117846,336122526~7073563,347366098~139655,349983031~6849925,357994231~2693797,375001040~3381344,381700185~7648272,391533675~33935,399456201~7300153] 2024-03-29T05:08:49.023 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~2135316,81195022~23151134,108145176~934353,114910182~5114642,120464135~4299057,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,256353754~7451407,269267167~7063560,298106400~3193945,304147414~1639746,313401620~414791,318129639~337373,323647203~3993690,333824662~117846,336122526~7073563,347366098~139655,349983031~6849925,357994231~2693797,375001040~3381344,381700185~7648272,391533675~33935,399456201~7300153] 2024-03-29T05:08:49.028 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~2135316,81195022~23151134,108145176~934353,114910182~9853010,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,256353754~7451407,268435456~7895271,298106400~3193945,301989888~3797272,313401620~414791,318129639~337373,323647203~3993690,332044929~1897579,336122526~7073563,347366098~139655,349983031~6849925,357994231~2693797,375001040~3381344,381700185~7648272,391533675~33935,399456201~7300153] 2024-03-29T05:08:49.029 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to 19 diff [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~5114642,120464135~4299057,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:49.029 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~5114642,120464135~4299057,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:49.034 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~9853010,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11151160,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:49.034 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to 15 diff [607349~4280051,33264680~4209092,50354796~1640367,84964601~734981,99411789~4934367,202433993~3601358,212065768~7948638,313401620~414791,381994856~6144098] 2024-03-29T05:08:49.034 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~4280051,33264680~4209092,50354796~1640367,84964601~734981,99411789~4934367,202433993~3601358,212065768~7948638,313401620~414791,381994856~6144098] 2024-03-29T05:08:49.039 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~4280051,33264680~4209092,50354796~1640367,84964601~734981,99411789~4934367,202433993~3601358,212065768~7948638,313401620~414791,381994856~6144098] 2024-03-29T05:08:49.039 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to 16 diff [607349~5934701,33264680~4209092,50354796~1640367,53365114~662488,81195022~6857628,99411789~4934367,108145176~934353,114910182~5114642,202433993~3601358,212065768~7948638,313401620~414791,323647203~3993690,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870] 2024-03-29T05:08:49.039 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,33264680~4209092,50354796~1640367,53365114~662488,81195022~6857628,99411789~4934367,108145176~934353,114910182~5114642,202433993~3601358,212065768~7948638,313401620~414791,323647203~3993690,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870] 2024-03-29T05:08:49.045 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,33264680~4209092,50354796~3672806,81195022~6857628,99411789~4934367,108145176~934353,114910182~5114642,202433993~3601358,212065768~7948638,313401620~414791,323647203~3993690,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870] 2024-03-29T05:08:49.045 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to 17 diff [607349~5934701,33264680~4209092,50354796~1640367,53365114~662488,81195022~6857628,88080139~7913273,99411789~4934367,108145176~934353,114910182~5114642,176445827~7860687,202433993~3601358,208104136~550405,211369000~8645406,256353754~7451407,269267167~7063560,313401620~414791,318129639~337373,323647203~3993690,336122526~7073563,347366098~139655,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870] 2024-03-29T05:08:49.045 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,33264680~4209092,50354796~1640367,53365114~662488,81195022~6857628,88080139~7913273,99411789~4934367,108145176~934353,114910182~5114642,176445827~7860687,202433993~3601358,208104136~550405,211369000~8645406,256353754~7451407,269267167~7063560,313401620~414791,318129639~337373,323647203~3993690,336122526~7073563,347366098~139655,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870] 2024-03-29T05:08:49.050 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,33264680~4209092,50354796~3672806,81195022~6857628,88080139~7913273,99411789~4934367,108145176~934353,114910182~5114642,176445827~7860687,202433993~3601358,208104136~550405,211369000~8645406,256353754~7451407,268435456~7895271,313401620~414791,318129639~337373,323647203~3993690,336122526~7073563,347366098~139655,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870] 2024-03-29T05:08:49.050 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to 18 diff [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~2135316,81195022~14798390,99411789~4934367,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,256353754~7451407,269267167~7063560,298106400~3193945,304147414~1639746,313401620~414791,318129639~337373,323647203~3993690,333824662~117846,336122526~7073563,347366098~139655,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870,405770812~985542] 2024-03-29T05:08:49.050 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~2135316,81195022~14798390,99411789~4934367,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,256353754~7451407,269267167~7063560,298106400~3193945,304147414~1639746,313401620~414791,318129639~337373,323647203~3993690,333824662~117846,336122526~7073563,347366098~139655,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870,405770812~985542] 2024-03-29T05:08:49.056 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~2135316,81195022~14798390,99411789~4934367,108145176~934353,114910182~9853010,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,256353754~7451407,268435456~7895271,298106400~3193945,301989888~3797272,313401620~414791,318129639~337373,323647203~3993690,332044929~1897579,336122526~7073563,347366098~139655,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870,405770812~985542] 2024-03-29T05:08:49.056 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to 19 diff [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~17399138,99411789~4934367,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:49.056 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~17399138,99411789~4934367,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:49.061 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~17399138,99411789~4934367,108145176~934353,114910182~9853010,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11151160,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:49.062 INFO:tasks.workunit.client.0.smithi181.stdout:from 15 to 16 diff [1721581~4820469,53365114~662488,81195022~6857628,108145176~934353,114910182~5114642,323647203~3993690,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870] 2024-03-29T05:08:49.062 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [1721581~4820469,53365114~662488,81195022~6857628,108145176~934353,114910182~5114642,323647203~3993690,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870] 2024-03-29T05:08:49.067 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [1721581~4820469,51995163~2032439,81195022~6857628,108145176~934353,114910182~5114642,323647203~3993690,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870] 2024-03-29T05:08:49.067 INFO:tasks.workunit.client.0.smithi181.stdout:from 15 to 17 diff [1721581~4820469,53365114~662488,81195022~6857628,88080139~7913273,108145176~934353,114910182~5114642,176445827~7860687,208104136~550405,211369000~1548772,256353754~7451407,269267167~7063560,318129639~337373,323647203~3993690,336122526~7073563,347366098~139655,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870] 2024-03-29T05:08:49.067 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [1721581~4820469,53365114~662488,81195022~6857628,88080139~7913273,108145176~934353,114910182~5114642,176445827~7860687,208104136~550405,211369000~1548772,256353754~7451407,269267167~7063560,318129639~337373,323647203~3993690,336122526~7073563,347366098~139655,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870] 2024-03-29T05:08:49.073 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [1721581~4820469,51995163~2032439,81195022~6857628,88080139~7913273,108145176~934353,114910182~5114642,176445827~7860687,208104136~550405,211369000~1548772,256353754~7451407,268435456~7895271,318129639~337373,323647203~3993690,336122526~7073563,347366098~139655,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870] 2024-03-29T05:08:49.073 INFO:tasks.workunit.client.0.smithi181.stdout:from 15 to 18 diff [1721581~4820469,46870117~7157485,59960135~3710142,78594274~2135316,81195022~14798390,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,256353754~7451407,269267167~7063560,298106400~3193945,304147414~1639746,318129639~337373,323647203~3993690,333824662~117846,336122526~7073563,347366098~139655,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870,405770812~985542] 2024-03-29T05:08:49.073 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [1721581~4820469,46870117~7157485,59960135~3710142,78594274~2135316,81195022~14798390,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,256353754~7451407,269267167~7063560,298106400~3193945,304147414~1639746,318129639~337373,323647203~3993690,333824662~117846,336122526~7073563,347366098~139655,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870,405770812~985542] 2024-03-29T05:08:49.079 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [1721581~4820469,46870117~7157485,59960135~3710142,78594274~2135316,81195022~14798390,108145176~934353,114910182~9853010,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,256353754~7451407,268435456~7895271,298106400~3193945,301989888~3797272,318129639~337373,323647203~3993690,332044929~1897579,336122526~7073563,347366098~139655,357994231~2693797,381700185~7648272,391533675~33935,399658726~3893870,405770812~985542] 2024-03-29T05:08:49.079 INFO:tasks.workunit.client.0.smithi181.stdout:from 15 to 19 diff [1721581~4820469,46870117~7157485,59960135~3710142,78594274~17399138,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:49.079 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [1721581~4820469,46870117~7157485,59960135~3710142,78594274~17399138,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:49.084 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [1721581~4820469,46870117~7157485,59960135~3710142,78594274~17399138,108145176~934353,114910182~9853010,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,318129639~337373,323647203~6256678,332044929~11151160,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:49.084 INFO:tasks.workunit.client.0.smithi181.stdout:from 16 to 17 diff [88080139~7913273,176445827~7860687,208104136~550405,211369000~1548772,256353754~7451407,269267167~7063560,318129639~337373,336122526~7073563,347366098~139655] 2024-03-29T05:08:49.084 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [88080139~7913273,176445827~7860687,208104136~550405,211369000~1548772,256353754~7451407,269267167~7063560,318129639~337373,336122526~7073563,347366098~139655] 2024-03-29T05:08:49.090 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [88080139~7913273,176445827~7860687,208104136~550405,211369000~1548772,256353754~7451407,268435456~7895271,318129639~337373,336122526~7073563,347366098~139655] 2024-03-29T05:08:49.090 INFO:tasks.workunit.client.0.smithi181.stdout:from 16 to 18 diff [46870117~7046614,59960135~3710142,78594274~2135316,85708102~10285310,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,256353754~7451407,269267167~7063560,298106400~3193945,304147414~1639746,318129639~337373,333824662~117846,336122526~7073563,347366098~139655,405770812~985542] 2024-03-29T05:08:49.090 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [46870117~7046614,59960135~3710142,78594274~2135316,85708102~10285310,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,256353754~7451407,269267167~7063560,298106400~3193945,304147414~1639746,318129639~337373,333824662~117846,336122526~7073563,347366098~139655,405770812~985542] 2024-03-29T05:08:49.096 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [46870117~7046614,59960135~3710142,78594274~2135316,85708102~10285310,120024824~4738368,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,256353754~7451407,268435456~7895271,298106400~3193945,301989888~3797272,318129639~337373,332044929~1897579,336122526~7073563,347366098~139655,405770812~985542] 2024-03-29T05:08:49.096 INFO:tasks.workunit.client.0.smithi181.stdout:from 16 to 19 diff [46870117~7046614,59960135~3710142,78594274~2722512,85708102~10285310,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,318129639~337373,324576755~5327126,333726986~9469103,347366098~139655,367522990~5006359,392541912~715499,398460268~2471818,405770812~985542] 2024-03-29T05:08:49.096 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [46870117~7046614,59960135~3710142,78594274~2722512,85708102~10285310,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,318129639~337373,324576755~5327126,333726986~9469103,347366098~139655,367522990~5006359,392541912~715499,398460268~2471818,405770812~985542] 2024-03-29T05:08:49.101 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [46870117~7046614,59960135~3710142,78594274~2722512,85708102~10285310,120024824~4738368,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,318129639~337373,324576755~5327126,332044929~11151160,347366098~139655,367522990~5006359,392541912~715499,398460268~2471818,405770812~985542] 2024-03-29T05:08:49.101 INFO:tasks.workunit.client.0.smithi181.stdout:from 17 to 18 diff [46870117~7046614,59960135~3710142,78594274~2135316,85708102~6172979,120464135~4299057,195709844~6886032,298106400~3193945,304147414~1639746,333824662~117846,405770812~985542] 2024-03-29T05:08:49.101 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [46870117~7046614,59960135~3710142,78594274~2135316,85708102~6172979,120464135~4299057,195709844~6886032,298106400~3193945,304147414~1639746,333824662~117846,405770812~985542] 2024-03-29T05:08:49.106 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [46870117~7046614,59960135~3710142,78594274~2135316,85708102~6172979,120024824~4738368,195709844~6886032,298106400~3193945,301989888~3797272,332044929~1897579,405770812~985542] 2024-03-29T05:08:49.107 INFO:tasks.workunit.client.0.smithi181.stdout:from 17 to 19 diff [46870117~7046614,59960135~3710142,78594274~2722512,85708102~6172979,120464135~4299057,195709844~6886032,251771759~3101503,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,324576755~5327126,333726986~3732200,367522990~5006359,392541912~715499,398460268~2471818,405770812~985542] 2024-03-29T05:08:49.107 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [46870117~7046614,59960135~3710142,78594274~2722512,85708102~6172979,120464135~4299057,195709844~6886032,251771759~3101503,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,324576755~5327126,333726986~3732200,367522990~5006359,392541912~715499,398460268~2471818,405770812~985542] 2024-03-29T05:08:49.112 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [46870117~7046614,59960135~3710142,78594274~2722512,85708102~6172979,120024824~4738368,195709844~6886032,251771759~3101503,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,324576755~5327126,332044929~5414257,367522990~5006359,392541912~715499,398460268~2471818,405770812~985542] 2024-03-29T05:08:49.112 INFO:tasks.workunit.client.0.smithi181.stdout:from 18 to 19 diff [80721595~595191,251771759~3101503,286060646~3118392,309777748~1877260,324576755~5327126,333726986~3732200,367522990~5006359,392541912~715499,398460268~2471818] 2024-03-29T05:08:49.112 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [80721595~595191,251771759~3101503,286060646~3118392,309777748~1877260,324576755~5327126,333726986~3732200,367522990~5006359,392541912~715499,398460268~2471818] 2024-03-29T05:08:49.118 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [80721595~595191,251771759~3101503,286060646~3118392,309777748~1877260,324576755~5327126,333726986~3732200,367522990~5006359,392541912~715499,398460268~2471818] 2024-03-29T05:08:49.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:49 smithi181 ceph-mon[101970]: failed to encode map e882 with expected crc 2024-03-29T05:08:49.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:49 smithi181 ceph-mon[101970]: osdmap e882: 8 total, 8 up, 8 in 2024-03-29T05:08:49.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:49 smithi181 ceph-mon[101970]: failed to encode map e882 with expected crc 2024-03-29T05:08:49.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:08:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:08:48] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.43.0" 2024-03-29T05:08:49.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:49 smithi181 ceph-mon[106466]: failed to encode map e882 with expected crc 2024-03-29T05:08:49.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:49 smithi181 ceph-mon[106466]: osdmap e882: 8 total, 8 up, 8 in 2024-03-29T05:08:49.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:49 smithi181 ceph-mon[106466]: failed to encode map e882 with expected crc 2024-03-29T05:08:49.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:49 smithi187 ceph-mon[93536]: failed to encode map e882 with expected crc 2024-03-29T05:08:49.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:49 smithi187 ceph-mon[93536]: osdmap e882: 8 total, 8 up, 8 in 2024-03-29T05:08:49.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:49 smithi187 ceph-mon[93536]: failed to encode map e882 with expected crc 2024-03-29T05:08:50.035 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:50.036 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:50.052 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:50.052 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:50.052 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:50.058 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:50.058 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:50.058 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:50.064 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:08:50.064 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:50.064 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:50.070 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:50.070 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:50.070 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:50.075 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:50.076 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:50.076 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:50.081 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,176445827~44178509,245550311~20244233,268435456~15049697,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:50.081 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:50.081 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:50.087 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~44178509,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:50.087 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:50.087 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:50.092 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:50.092 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:50.093 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:50.098 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:50.098 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:50.098 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:50.104 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:50.104 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:50.104 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:50.109 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:50.109 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:50.109 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:50.115 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,323647203~6256678,332044929~11341135,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:50.115 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:50.115 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:50.120 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11341135,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:50.120 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to HEAD diff [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~5114642,120464135~4299057,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:50.120 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~5114642,120464135~4299057,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:50.125 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~9853010,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11151160,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:50.126 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to HEAD diff [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~17399138,99411789~4934367,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:50.126 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~17399138,99411789~4934367,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:50.131 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~17399138,99411789~4934367,108145176~934353,114910182~9853010,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11151160,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:50.131 INFO:tasks.workunit.client.0.smithi181.stdout:from 15 to HEAD diff [1721581~4820469,46870117~7157485,59960135~3710142,78594274~17399138,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:50.131 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [1721581~4820469,46870117~7157485,59960135~3710142,78594274~17399138,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:50.137 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [1721581~4820469,46870117~7157485,59960135~3710142,78594274~17399138,108145176~934353,114910182~9853010,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,318129639~337373,323647203~6256678,332044929~11151160,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:50.137 INFO:tasks.workunit.client.0.smithi181.stdout:from 16 to HEAD diff [46870117~7046614,59960135~3710142,78594274~2722512,85708102~10285310,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,318129639~337373,324576755~5327126,333726986~9469103,347366098~139655,367522990~5006359,392541912~715499,398460268~2471818,405770812~985542] 2024-03-29T05:08:50.137 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [46870117~7046614,59960135~3710142,78594274~2722512,85708102~10285310,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,318129639~337373,324576755~5327126,333726986~9469103,347366098~139655,367522990~5006359,392541912~715499,398460268~2471818,405770812~985542] 2024-03-29T05:08:50.142 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [46870117~7046614,59960135~3710142,78594274~2722512,85708102~10285310,120024824~4738368,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,318129639~337373,324576755~5327126,332044929~11151160,347366098~139655,367522990~5006359,392541912~715499,398460268~2471818,405770812~985542] 2024-03-29T05:08:50.143 INFO:tasks.workunit.client.0.smithi181.stdout:from 17 to HEAD diff [46870117~7046614,59960135~3710142,78594274~2722512,85708102~6172979,120464135~4299057,195709844~6886032,251771759~3101503,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,324576755~5327126,333726986~3732200,367522990~5006359,392541912~715499,398460268~2471818,405770812~985542] 2024-03-29T05:08:50.143 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [46870117~7046614,59960135~3710142,78594274~2722512,85708102~6172979,120464135~4299057,195709844~6886032,251771759~3101503,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,324576755~5327126,333726986~3732200,367522990~5006359,392541912~715499,398460268~2471818,405770812~985542] 2024-03-29T05:08:50.148 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [46870117~7046614,59960135~3710142,78594274~2722512,85708102~6172979,120024824~4738368,195709844~6886032,251771759~3101503,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,324576755~5327126,332044929~5414257,367522990~5006359,392541912~715499,398460268~2471818,405770812~985542] 2024-03-29T05:08:50.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:50 smithi181 ceph-mon[106466]: failed to encode map e882 with expected crc 2024-03-29T05:08:50.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:50 smithi181 ceph-mon[106466]: pgmap v1051: 105 pgs: 105 active+clean; 851 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.2 KiB/s rd, 39 MiB/s wr, 19 op/s 2024-03-29T05:08:50.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:50 smithi181 ceph-mon[101970]: failed to encode map e882 with expected crc 2024-03-29T05:08:50.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:50 smithi181 ceph-mon[101970]: pgmap v1051: 105 pgs: 105 active+clean; 851 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.2 KiB/s rd, 39 MiB/s wr, 19 op/s 2024-03-29T05:08:50.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:50 smithi187 ceph-mon[93536]: failed to encode map e882 with expected crc 2024-03-29T05:08:50.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:50 smithi187 ceph-mon[93536]: pgmap v1051: 105 pgs: 105 active+clean; 851 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.2 KiB/s rd, 39 MiB/s wr, 19 op/s 2024-03-29T05:08:51.046 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:51.047 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:51.060 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:51.060 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:51.060 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:51.066 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:51.066 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:51.066 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:51.072 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:08:51.073 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:51.073 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:51.079 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:51.079 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:51.079 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:51.084 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:51.084 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:51.085 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:51.090 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,176445827~44178509,245550311~20244233,268435456~15049697,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:51.090 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:51.090 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:51.096 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~44178509,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:51.096 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:51.096 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:51.101 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:51.101 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:51.102 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:51.107 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:51.107 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:51.107 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:51.113 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:51.113 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:51.113 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:51.118 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:51.118 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:51.119 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:51.124 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,323647203~6256678,332044929~11341135,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:51.124 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:51.124 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:51.130 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11341135,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:51.130 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to HEAD diff [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~5114642,120464135~4299057,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:51.130 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~5114642,120464135~4299057,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:51.135 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~9853010,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11151160,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:51.135 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to HEAD diff [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~17399138,99411789~4934367,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:51.136 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~17399138,99411789~4934367,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:51.141 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~17399138,99411789~4934367,108145176~934353,114910182~9853010,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11151160,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:51.141 INFO:tasks.workunit.client.0.smithi181.stdout:from 15 to HEAD diff [1721581~4820469,46870117~7157485,59960135~3710142,78594274~17399138,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:51.141 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [1721581~4820469,46870117~7157485,59960135~3710142,78594274~17399138,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:51.146 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [1721581~4820469,46870117~7157485,59960135~3710142,78594274~17399138,108145176~934353,114910182~9853010,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,318129639~337373,323647203~6256678,332044929~11151160,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:51.147 INFO:tasks.workunit.client.0.smithi181.stdout:from 16 to HEAD diff [46870117~7046614,59960135~3710142,78594274~2722512,85708102~10285310,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,318129639~337373,324576755~5327126,333726986~9469103,347366098~139655,367522990~5006359,392541912~715499,398460268~2471818,405770812~985542] 2024-03-29T05:08:51.147 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [46870117~7046614,59960135~3710142,78594274~2722512,85708102~10285310,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,318129639~337373,324576755~5327126,333726986~9469103,347366098~139655,367522990~5006359,392541912~715499,398460268~2471818,405770812~985542] 2024-03-29T05:08:51.152 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [46870117~7046614,59960135~3710142,78594274~2722512,85708102~10285310,120024824~4738368,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,318129639~337373,324576755~5327126,332044929~11151160,347366098~139655,367522990~5006359,392541912~715499,398460268~2471818,405770812~985542] 2024-03-29T05:08:51.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:51 smithi187 ceph-mon[93536]: failed to encode map e883 with expected crc 2024-03-29T05:08:51.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:51 smithi187 ceph-mon[93536]: osdmap e883: 8 total, 8 up, 8 in 2024-03-29T05:08:51.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:51 smithi187 ceph-mon[93536]: failed to encode map e883 with expected crc 2024-03-29T05:08:51.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:51 smithi187 ceph-mon[93536]: failed to encode map e883 with expected crc 2024-03-29T05:08:51.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:51 smithi181 ceph-mon[101970]: failed to encode map e883 with expected crc 2024-03-29T05:08:51.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:51 smithi181 ceph-mon[101970]: osdmap e883: 8 total, 8 up, 8 in 2024-03-29T05:08:51.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:51 smithi181 ceph-mon[101970]: failed to encode map e883 with expected crc 2024-03-29T05:08:51.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:51 smithi181 ceph-mon[101970]: failed to encode map e883 with expected crc 2024-03-29T05:08:51.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:51 smithi181 ceph-mon[106466]: failed to encode map e883 with expected crc 2024-03-29T05:08:51.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:51 smithi181 ceph-mon[106466]: osdmap e883: 8 total, 8 up, 8 in 2024-03-29T05:08:51.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:51 smithi181 ceph-mon[106466]: failed to encode map e883 with expected crc 2024-03-29T05:08:51.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:51 smithi181 ceph-mon[106466]: failed to encode map e883 with expected crc 2024-03-29T05:08:52.058 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:52.059 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:52.073 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:52.073 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:52.074 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:52.079 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:52.079 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:52.080 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:52.086 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:08:52.086 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:52.086 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:52.092 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:52.092 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:52.092 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:52.097 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:52.098 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:52.098 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:52.103 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,176445827~44178509,245550311~20244233,268435456~15049697,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:52.103 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:52.103 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:52.109 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~44178509,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:52.109 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:52.109 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:52.115 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:52.115 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:52.115 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:52.121 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:52.121 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:52.121 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:52.126 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:52.126 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:52.127 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:52.132 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:52.132 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:52.132 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:52.138 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,323647203~6256678,332044929~11341135,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:52.138 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:52.138 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:52.143 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11341135,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:52.143 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to HEAD diff [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~5114642,120464135~4299057,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:52.144 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~5114642,120464135~4299057,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:52.149 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~9853010,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11151160,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:52.149 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to HEAD diff [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~17399138,99411789~4934367,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:52.149 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~17399138,99411789~4934367,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:52.154 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~17399138,99411789~4934367,108145176~934353,114910182~9853010,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11151160,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:52.154 INFO:tasks.workunit.client.0.smithi181.stdout:from 15 to HEAD diff [1721581~4820469,46870117~7157485,59960135~3710142,78594274~17399138,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:52.154 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [1721581~4820469,46870117~7157485,59960135~3710142,78594274~17399138,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:52.159 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [1721581~4820469,46870117~7157485,59960135~3710142,78594274~17399138,108145176~934353,114910182~9853010,176445827~7860687,195709844~6886032,208104136~550405,211369000~1548772,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,318129639~337373,323647203~6256678,332044929~11151160,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:52.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:52 smithi187 ceph-mon[93536]: pgmap v1053: 105 pgs: 105 active+clean; 877 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 211 KiB/s rd, 42 MiB/s wr, 598 op/s 2024-03-29T05:08:52.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:52 smithi187 ceph-mon[93536]: failed to encode map e884 with expected crc 2024-03-29T05:08:52.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:52 smithi187 ceph-mon[93536]: failed to encode map e884 with expected crc 2024-03-29T05:08:52.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:52 smithi187 ceph-mon[93536]: osdmap e884: 8 total, 8 up, 8 in 2024-03-29T05:08:52.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:52 smithi187 ceph-mon[93536]: failed to encode map e884 with expected crc 2024-03-29T05:08:52.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:52 smithi181 ceph-mon[101970]: pgmap v1053: 105 pgs: 105 active+clean; 877 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 211 KiB/s rd, 42 MiB/s wr, 598 op/s 2024-03-29T05:08:52.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:52 smithi181 ceph-mon[101970]: failed to encode map e884 with expected crc 2024-03-29T05:08:52.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:52 smithi181 ceph-mon[101970]: failed to encode map e884 with expected crc 2024-03-29T05:08:52.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:52 smithi181 ceph-mon[101970]: osdmap e884: 8 total, 8 up, 8 in 2024-03-29T05:08:52.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:52 smithi181 ceph-mon[101970]: failed to encode map e884 with expected crc 2024-03-29T05:08:52.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:52 smithi181 ceph-mon[106466]: pgmap v1053: 105 pgs: 105 active+clean; 877 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 211 KiB/s rd, 42 MiB/s wr, 598 op/s 2024-03-29T05:08:52.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:52 smithi181 ceph-mon[106466]: failed to encode map e884 with expected crc 2024-03-29T05:08:52.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:52 smithi181 ceph-mon[106466]: failed to encode map e884 with expected crc 2024-03-29T05:08:52.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:52 smithi181 ceph-mon[106466]: osdmap e884: 8 total, 8 up, 8 in 2024-03-29T05:08:52.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:52 smithi181 ceph-mon[106466]: failed to encode map e884 with expected crc 2024-03-29T05:08:53.073 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:53.073 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:53.087 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:53.087 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:53.088 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:53.094 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:53.094 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:53.094 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:53.100 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:08:53.100 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:53.100 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:53.106 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:53.106 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:53.106 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:53.112 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:53.112 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:53.112 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:53.117 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,176445827~44178509,245550311~20244233,268435456~15049697,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:53.117 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:53.117 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:53.123 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~44178509,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:53.123 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:53.123 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:53.129 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:53.129 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:53.129 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:53.134 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:53.135 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:53.135 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:53.140 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:53.140 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:53.140 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:53.146 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:53.146 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:53.146 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:53.151 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,323647203~6256678,332044929~11341135,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:53.152 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:53.152 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:53.157 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11341135,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:53.157 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to HEAD diff [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~5114642,120464135~4299057,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:53.157 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~5114642,120464135~4299057,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:53.162 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~9853010,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11151160,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:53.162 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to HEAD diff [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~17399138,99411789~4934367,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:53.163 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~17399138,99411789~4934367,108145176~934353,114910182~5114642,120464135~4299057,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:53.168 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,33264680~4209092,46870117~7157485,59960135~3710142,78594274~17399138,99411789~4934367,108145176~934353,114910182~9853010,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11151160,347366098~139655,357994231~2693797,367522990~5006359,381700185~7648272,391533675~33935,392541912~715499,398460268~5092328,405770812~985542] 2024-03-29T05:08:53.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:53 smithi187 ceph-mon[93536]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:53 smithi187 ceph-mon[93536]: osdmap e885: 8 total, 8 up, 8 in 2024-03-29T05:08:53.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:53 smithi187 ceph-mon[93536]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:53 smithi187 ceph-mon[93536]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:53 smithi187 ceph-mon[93536]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:53 smithi187 ceph-mon[93536]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:53 smithi187 ceph-mon[93536]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:53 smithi187 ceph-mon[93536]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:53 smithi181 ceph-mon[101970]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:53 smithi181 ceph-mon[101970]: osdmap e885: 8 total, 8 up, 8 in 2024-03-29T05:08:53.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:53 smithi181 ceph-mon[101970]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:53 smithi181 ceph-mon[101970]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:53 smithi181 ceph-mon[101970]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:53 smithi181 ceph-mon[101970]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:53 smithi181 ceph-mon[101970]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:53 smithi181 ceph-mon[101970]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:53 smithi181 ceph-mon[106466]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:53 smithi181 ceph-mon[106466]: osdmap e885: 8 total, 8 up, 8 in 2024-03-29T05:08:53.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:53 smithi181 ceph-mon[106466]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:53 smithi181 ceph-mon[106466]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:53 smithi181 ceph-mon[106466]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:53 smithi181 ceph-mon[106466]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:53 smithi181 ceph-mon[106466]: failed to encode map e885 with expected crc 2024-03-29T05:08:53.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:53 smithi181 ceph-mon[106466]: failed to encode map e885 with expected crc 2024-03-29T05:08:54.066 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:54.066 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:54.090 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:54.091 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:54.091 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:54.096 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:54.096 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:54.097 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:54.102 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:08:54.102 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:54.102 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:54.108 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:54.108 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:54.108 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:54.114 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:54.114 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:54.114 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:54.119 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,176445827~44178509,245550311~20244233,268435456~15049697,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:54.120 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:54.120 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:54.125 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~44178509,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:54.125 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:54.125 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:54.131 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:54.131 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:54.131 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:54.136 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:54.137 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:54.137 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:54.142 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:54.142 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:54.143 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:54.148 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:54.148 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:54.148 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:54.153 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,323647203~6256678,332044929~11341135,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:54.153 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:54.154 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:54.159 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11341135,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:54.159 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to HEAD diff [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~5114642,120464135~4299057,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:54.159 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~5114642,120464135~4299057,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9469103,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:54.164 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,27077192~10396580,46870117~7157485,59746300~3923977,78594274~25751882,108145176~934353,114910182~9853010,143272389~3772807,176445827~7860687,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11151160,347366098~139655,349983031~6849925,357994231~2693797,367522990~5006359,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,398460268~8296086] 2024-03-29T05:08:54.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:54 smithi187 ceph-mon[93536]: pgmap v1056: 105 pgs: 105 active+clean; 882 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 243 KiB/s rd, 8.5 MiB/s wr, 4.70k op/s 2024-03-29T05:08:54.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:54 smithi187 ceph-mon[93536]: failed to encode map e886 with expected crc 2024-03-29T05:08:54.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:54 smithi187 ceph-mon[93536]: failed to encode map e886 with expected crc 2024-03-29T05:08:54.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:54 smithi187 ceph-mon[93536]: failed to encode map e886 with expected crc 2024-03-29T05:08:54.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:54 smithi187 ceph-mon[93536]: failed to encode map e886 with expected crc 2024-03-29T05:08:54.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:54 smithi187 ceph-mon[93536]: failed to encode map e886 with expected crc 2024-03-29T05:08:54.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:54 smithi187 ceph-mon[93536]: osdmap e886: 8 total, 8 up, 8 in 2024-03-29T05:08:54.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:54 smithi181 ceph-mon[106466]: pgmap v1056: 105 pgs: 105 active+clean; 882 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 243 KiB/s rd, 8.5 MiB/s wr, 4.70k op/s 2024-03-29T05:08:54.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:54 smithi181 ceph-mon[106466]: failed to encode map e886 with expected crc 2024-03-29T05:08:54.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:54 smithi181 ceph-mon[106466]: failed to encode map e886 with expected crc 2024-03-29T05:08:54.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:54 smithi181 ceph-mon[106466]: failed to encode map e886 with expected crc 2024-03-29T05:08:54.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:54 smithi181 ceph-mon[106466]: failed to encode map e886 with expected crc 2024-03-29T05:08:54.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:54 smithi181 ceph-mon[106466]: failed to encode map e886 with expected crc 2024-03-29T05:08:54.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:54 smithi181 ceph-mon[106466]: osdmap e886: 8 total, 8 up, 8 in 2024-03-29T05:08:54.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:54 smithi181 ceph-mon[101970]: pgmap v1056: 105 pgs: 105 active+clean; 882 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 243 KiB/s rd, 8.5 MiB/s wr, 4.70k op/s 2024-03-29T05:08:54.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:54 smithi181 ceph-mon[101970]: failed to encode map e886 with expected crc 2024-03-29T05:08:54.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:54 smithi181 ceph-mon[101970]: failed to encode map e886 with expected crc 2024-03-29T05:08:54.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:54 smithi181 ceph-mon[101970]: failed to encode map e886 with expected crc 2024-03-29T05:08:54.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:54 smithi181 ceph-mon[101970]: failed to encode map e886 with expected crc 2024-03-29T05:08:54.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:54 smithi181 ceph-mon[101970]: failed to encode map e886 with expected crc 2024-03-29T05:08:54.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:54 smithi181 ceph-mon[101970]: osdmap e886: 8 total, 8 up, 8 in 2024-03-29T05:08:55.087 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:55.087 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:55.102 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:55.102 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:55.102 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:55.110 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:55.111 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:55.111 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:55.113 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:08:55.113 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:55.113 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:55.119 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:55.119 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:55.119 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:55.124 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:55.125 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:55.125 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:55.130 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,176445827~44178509,245550311~20244233,268435456~15049697,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:55.131 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:55.131 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:55.136 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~44178509,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:55.136 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:55.136 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:55.142 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:55.142 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:55.142 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:55.148 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:55.148 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:55.148 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:55.153 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:55.153 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:55.153 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:55.159 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:55.159 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:55.159 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:55.165 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,323647203~6256678,332044929~11341135,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:55.165 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:55.165 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,269267167~7063560,286060646~3118392,298106400~3193945,304147414~1639746,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:55.170 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~3101503,256353754~7451407,268435456~7895271,286060646~3118392,298106400~3193945,301989888~3797272,309777748~1877260,313401620~414791,318129639~337373,323647203~6256678,332044929~11341135,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:55.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:55 smithi187 ceph-mon[93536]: osdmap e887: 8 total, 8 up, 8 in 2024-03-29T05:08:55.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:55 smithi187 ceph-mon[93536]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:55 smithi187 ceph-mon[93536]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:55 smithi187 ceph-mon[93536]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:55 smithi187 ceph-mon[93536]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:55 smithi187 ceph-mon[93536]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:55 smithi187 ceph-mon[93536]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:55 smithi187 ceph-mon[93536]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:55 smithi187 ceph-mon[93536]: pgmap v1059: 105 pgs: 105 active+clean; 880 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 164 KiB/s rd, 2.5 MiB/s wr, 5.83k op/s 2024-03-29T05:08:55.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[101970]: osdmap e887: 8 total, 8 up, 8 in 2024-03-29T05:08:55.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[101970]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[101970]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[101970]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[101970]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[101970]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[101970]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[101970]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[101970]: pgmap v1059: 105 pgs: 105 active+clean; 880 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 164 KiB/s rd, 2.5 MiB/s wr, 5.83k op/s 2024-03-29T05:08:55.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[106466]: osdmap e887: 8 total, 8 up, 8 in 2024-03-29T05:08:55.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[106466]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[106466]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[106466]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[106466]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[106466]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[106466]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[106466]: failed to encode map e887 with expected crc 2024-03-29T05:08:55.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:55 smithi181 ceph-mon[106466]: pgmap v1059: 105 pgs: 105 active+clean; 880 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 164 KiB/s rd, 2.5 MiB/s wr, 5.83k op/s 2024-03-29T05:08:56.089 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:56.133 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:56.136 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:56.136 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:56.136 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:56.136 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:56.136 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:56.136 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:56.136 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:08:56.136 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:56.137 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:56.137 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:56.137 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:56.137 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:56.137 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:56.137 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:56.137 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:56.137 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,176445827~44178509,245550311~20244233,268435456~15049697,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:56.137 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:56.137 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:56.142 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~44178509,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:56.142 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:56.142 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:56.147 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:56.147 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:56.147 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:56.153 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:56.153 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:56.153 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:56.158 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:56.159 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:56.159 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:56.164 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:56.164 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:56.164 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,323647203~6256678,333726986~9659078,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:56.169 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,176445827~12680281,195709844~10325507,208104136~550405,211369000~8645406,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,323647203~6256678,332044929~11341135,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~7648272,391533675~33935,392541912~715499,394051856~4008344,398460268~8296086] 2024-03-29T05:08:56.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:56 smithi187 ceph-mon[93536]: failed to encode map e888 with expected crc 2024-03-29T05:08:56.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:56 smithi187 ceph-mon[93536]: failed to encode map e888 with expected crc 2024-03-29T05:08:56.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:56 smithi187 ceph-mon[93536]: failed to encode map e888 with expected crc 2024-03-29T05:08:56.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:56 smithi187 ceph-mon[93536]: failed to encode map e888 with expected crc 2024-03-29T05:08:56.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:56 smithi187 ceph-mon[93536]: failed to encode map e888 with expected crc 2024-03-29T05:08:56.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:56 smithi187 ceph-mon[93536]: osdmap e888: 8 total, 8 up, 8 in 2024-03-29T05:08:56.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:56 smithi181 ceph-mon[106466]: failed to encode map e888 with expected crc 2024-03-29T05:08:56.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:56 smithi181 ceph-mon[106466]: failed to encode map e888 with expected crc 2024-03-29T05:08:56.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:56 smithi181 ceph-mon[106466]: failed to encode map e888 with expected crc 2024-03-29T05:08:56.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:56 smithi181 ceph-mon[106466]: failed to encode map e888 with expected crc 2024-03-29T05:08:56.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:56 smithi181 ceph-mon[106466]: failed to encode map e888 with expected crc 2024-03-29T05:08:56.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:56 smithi181 ceph-mon[106466]: osdmap e888: 8 total, 8 up, 8 in 2024-03-29T05:08:56.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:56 smithi181 ceph-mon[101970]: failed to encode map e888 with expected crc 2024-03-29T05:08:56.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:56 smithi181 ceph-mon[101970]: failed to encode map e888 with expected crc 2024-03-29T05:08:56.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:56 smithi181 ceph-mon[101970]: failed to encode map e888 with expected crc 2024-03-29T05:08:56.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:56 smithi181 ceph-mon[101970]: failed to encode map e888 with expected crc 2024-03-29T05:08:56.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:56 smithi181 ceph-mon[101970]: failed to encode map e888 with expected crc 2024-03-29T05:08:56.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:56 smithi181 ceph-mon[101970]: osdmap e888: 8 total, 8 up, 8 in 2024-03-29T05:08:57.092 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:57.092 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:57.115 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:57.115 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:57.115 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:57.120 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:57.120 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:57.120 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:57.126 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:08:57.126 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:57.126 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:57.132 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:57.132 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:57.132 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:57.137 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:57.137 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.137 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.143 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,176445827~44178509,245550311~20244233,268435456~15049697,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.143 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.143 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.149 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~44178509,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.149 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.149 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.154 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.154 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:57.154 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:57.160 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:57.160 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:57.160 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:57.165 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:57.165 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~5114642,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,298106400~3193945,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:57.171 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,46870117~7157485,57640836~6971573,66741740~1785033,78594274~25751882,108145176~934353,109482434~4861457,114910182~9853010,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,298106400~3193945,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~6671873,375001040~3381344,381700185~16360015,398460268~8296086] 2024-03-29T05:08:57.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:57 smithi187 ceph-mon[93536]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:57 smithi187 ceph-mon[93536]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:57 smithi187 ceph-mon[93536]: osdmap e889: 8 total, 8 up, 8 in 2024-03-29T05:08:57.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:57 smithi187 ceph-mon[93536]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:57 smithi187 ceph-mon[93536]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:57 smithi187 ceph-mon[93536]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:57 smithi187 ceph-mon[93536]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:57 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:08:57.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:57 smithi187 ceph-mon[93536]: pgmap v1062: 105 pgs: 105 active+clean; 880 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 129 KiB/s rd, 6.0 KiB/s wr, 1.43k op/s 2024-03-29T05:08:57.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[106466]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[106466]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[106466]: osdmap e889: 8 total, 8 up, 8 in 2024-03-29T05:08:57.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[106466]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[106466]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[106466]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[106466]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:08:57.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[106466]: pgmap v1062: 105 pgs: 105 active+clean; 880 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 129 KiB/s rd, 6.0 KiB/s wr, 1.43k op/s 2024-03-29T05:08:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[101970]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[101970]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[101970]: osdmap e889: 8 total, 8 up, 8 in 2024-03-29T05:08:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[101970]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[101970]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[101970]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[101970]: failed to encode map e889 with expected crc 2024-03-29T05:08:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:08:57.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:57 smithi181 ceph-mon[101970]: pgmap v1062: 105 pgs: 105 active+clean; 880 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 129 KiB/s rd, 6.0 KiB/s wr, 1.43k op/s 2024-03-29T05:08:57.717 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:57.718 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:57.738 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:57.738 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:57.738 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:57.744 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:57.744 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:57.744 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:57.749 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:08:57.749 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:57.749 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:57.755 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:57.755 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:57.755 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:57.760 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:57.760 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.761 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.766 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,176445827~44178509,245550311~20244233,268435456~15049697,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.766 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.769 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.772 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~44178509,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.772 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.772 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.777 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:57.777 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:57.777 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:57.785 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:57.785 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:57.785 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:57.790 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,45179185~8848417,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143272389~3772807,147439418~4104761,176445827~12680281,189795676~16239675,208104136~550405,211369000~9255336,251771759~12033402,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~16360015,398460268~8296086,407807916~5424201] 2024-03-29T05:08:58.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:58 smithi187 ceph-mon[93536]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:58 smithi187 ceph-mon[93536]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:58 smithi187 ceph-mon[93536]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:58 smithi187 ceph-mon[93536]: osdmap e890: 8 total, 8 up, 8 in 2024-03-29T05:08:58.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:58 smithi187 ceph-mon[93536]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:58 smithi187 ceph-mon[93536]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:58 smithi187 ceph-mon[93536]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:58 smithi187 ceph-mon[93536]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:58 smithi187 ceph-mon[93536]: failed to encode map e891 with expected crc 2024-03-29T05:08:58.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:58 smithi187 ceph-mon[93536]: failed to encode map e891 with expected crc 2024-03-29T05:08:58.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:58 smithi187 ceph-mon[93536]: osdmap e891: 8 total, 8 up, 8 in 2024-03-29T05:08:58.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:58 smithi187 ceph-mon[93536]: failed to encode map e891 with expected crc 2024-03-29T05:08:58.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:58 smithi187 ceph-mon[93536]: failed to encode map e891 with expected crc 2024-03-29T05:08:58.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[106466]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[106466]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[106466]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[106466]: osdmap e890: 8 total, 8 up, 8 in 2024-03-29T05:08:58.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[106466]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[106466]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[106466]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[106466]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[106466]: failed to encode map e891 with expected crc 2024-03-29T05:08:58.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[106466]: failed to encode map e891 with expected crc 2024-03-29T05:08:58.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[106466]: osdmap e891: 8 total, 8 up, 8 in 2024-03-29T05:08:58.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[106466]: failed to encode map e891 with expected crc 2024-03-29T05:08:58.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[106466]: failed to encode map e891 with expected crc 2024-03-29T05:08:58.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[101970]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[101970]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[101970]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[101970]: osdmap e890: 8 total, 8 up, 8 in 2024-03-29T05:08:58.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[101970]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[101970]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[101970]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[101970]: failed to encode map e890 with expected crc 2024-03-29T05:08:58.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[101970]: failed to encode map e891 with expected crc 2024-03-29T05:08:58.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[101970]: failed to encode map e891 with expected crc 2024-03-29T05:08:58.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[101970]: osdmap e891: 8 total, 8 up, 8 in 2024-03-29T05:08:58.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[101970]: failed to encode map e891 with expected crc 2024-03-29T05:08:58.456 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-mon[101970]: failed to encode map e891 with expected crc 2024-03-29T05:08:58.721 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:58.721 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:58.742 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:58.742 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:58.742 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:58.748 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:58.748 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:58.748 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:58.753 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:08:58.754 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:58.754 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:58.759 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:58.759 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:58.759 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:58.765 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:58.765 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:58.765 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:58.771 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,176445827~44178509,245550311~20244233,268435456~15049697,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:58.771 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:58.772 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:58.777 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~44178509,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:58.777 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:58.777 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:58.782 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:58.782 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:58.783 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,18311766~6078807,27077192~10396580,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~10542390,120464135~4299057,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:58.788 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,18311766~6078807,25165824~12307948,39741677~14285925,57640836~6971573,66741740~1785033,72073222~2361176,78594274~25751882,108145176~934353,109482434~15280758,126744492~8413924,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25056169,407807916~5424201] 2024-03-29T05:08:59.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:08:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:08:58] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.43.0" 2024-03-29T05:08:59.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:59 smithi181 ceph-mon[106466]: failed to encode map e891 with expected crc 2024-03-29T05:08:59.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:59 smithi181 ceph-mon[106466]: failed to encode map e891 with expected crc 2024-03-29T05:08:59.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:59 smithi181 ceph-mon[106466]: failed to encode map e891 with expected crc 2024-03-29T05:08:59.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:59 smithi181 ceph-mon[106466]: pgmap v1065: 105 pgs: 105 active+clean; 826 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.7 KiB/s rd, 0 B/s wr, 1.28k op/s 2024-03-29T05:08:59.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:59 smithi181 ceph-mon[106466]: failed to encode map e892 with expected crc 2024-03-29T05:08:59.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:59 smithi181 ceph-mon[106466]: failed to encode map e892 with expected crc 2024-03-29T05:08:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:59 smithi181 ceph-mon[106466]: failed to encode map e892 with expected crc 2024-03-29T05:08:59.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:08:59 smithi181 ceph-mon[106466]: osdmap e892: 8 total, 8 up, 8 in 2024-03-29T05:08:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:59 smithi181 ceph-mon[101970]: failed to encode map e891 with expected crc 2024-03-29T05:08:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:59 smithi181 ceph-mon[101970]: failed to encode map e891 with expected crc 2024-03-29T05:08:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:59 smithi181 ceph-mon[101970]: failed to encode map e891 with expected crc 2024-03-29T05:08:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:59 smithi181 ceph-mon[101970]: pgmap v1065: 105 pgs: 105 active+clean; 826 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.7 KiB/s rd, 0 B/s wr, 1.28k op/s 2024-03-29T05:08:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:59 smithi181 ceph-mon[101970]: failed to encode map e892 with expected crc 2024-03-29T05:08:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:59 smithi181 ceph-mon[101970]: failed to encode map e892 with expected crc 2024-03-29T05:08:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:59 smithi181 ceph-mon[101970]: failed to encode map e892 with expected crc 2024-03-29T05:08:59.205 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:08:59 smithi181 ceph-mon[101970]: osdmap e892: 8 total, 8 up, 8 in 2024-03-29T05:08:59.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:59 smithi187 ceph-mon[93536]: failed to encode map e891 with expected crc 2024-03-29T05:08:59.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:59 smithi187 ceph-mon[93536]: failed to encode map e891 with expected crc 2024-03-29T05:08:59.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:59 smithi187 ceph-mon[93536]: failed to encode map e891 with expected crc 2024-03-29T05:08:59.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:59 smithi187 ceph-mon[93536]: pgmap v1065: 105 pgs: 105 active+clean; 826 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.7 KiB/s rd, 0 B/s wr, 1.28k op/s 2024-03-29T05:08:59.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:59 smithi187 ceph-mon[93536]: failed to encode map e892 with expected crc 2024-03-29T05:08:59.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:59 smithi187 ceph-mon[93536]: failed to encode map e892 with expected crc 2024-03-29T05:08:59.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:59 smithi187 ceph-mon[93536]: failed to encode map e892 with expected crc 2024-03-29T05:08:59.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:08:59 smithi187 ceph-mon[93536]: osdmap e892: 8 total, 8 up, 8 in 2024-03-29T05:08:59.716 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:59.716 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:59.740 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:59.740 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:59.740 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:59.745 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:08:59.746 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:59.746 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:08:59.751 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:08:59.751 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:59.751 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:59.757 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:59.757 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:59.757 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:59.763 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:08:59.763 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:59.763 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:59.768 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,176445827~44178509,245550311~20244233,268435456~15049697,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:59.768 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:59.768 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:59.774 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~44178509,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:59.774 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:59.774 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~9983177,318129639~337373,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:08:59.779 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57640836~6971573,65183907~3342866,72073222~2361176,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~12680281,189795676~16239675,207668912~12955424,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~11718094,318129639~337373,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:09:00.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:00 smithi187 ceph-mon[93536]: failed to encode map e892 with expected crc 2024-03-29T05:09:00.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:00 smithi187 ceph-mon[93536]: failed to encode map e892 with expected crc 2024-03-29T05:09:00.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:00 smithi187 ceph-mon[93536]: failed to encode map e892 with expected crc 2024-03-29T05:09:00.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:00 smithi187 ceph-mon[93536]: failed to encode map e892 with expected crc 2024-03-29T05:09:00.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:00 smithi187 ceph-mon[93536]: failed to encode map e892 with expected crc 2024-03-29T05:09:00.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:00 smithi187 ceph-mon[93536]: osdmap e893: 8 total, 8 up, 8 in 2024-03-29T05:09:00.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:00 smithi187 ceph-mon[93536]: failed to encode map e893 with expected crc 2024-03-29T05:09:00.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:00 smithi187 ceph-mon[93536]: failed to encode map e893 with expected crc 2024-03-29T05:09:00.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:00 smithi181 ceph-mon[106466]: failed to encode map e892 with expected crc 2024-03-29T05:09:00.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:00 smithi181 ceph-mon[106466]: failed to encode map e892 with expected crc 2024-03-29T05:09:00.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:00 smithi181 ceph-mon[106466]: failed to encode map e892 with expected crc 2024-03-29T05:09:00.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:00 smithi181 ceph-mon[106466]: failed to encode map e892 with expected crc 2024-03-29T05:09:00.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:00 smithi181 ceph-mon[106466]: failed to encode map e892 with expected crc 2024-03-29T05:09:00.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:00 smithi181 ceph-mon[106466]: osdmap e893: 8 total, 8 up, 8 in 2024-03-29T05:09:00.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:00 smithi181 ceph-mon[106466]: failed to encode map e893 with expected crc 2024-03-29T05:09:00.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:00 smithi181 ceph-mon[106466]: failed to encode map e893 with expected crc 2024-03-29T05:09:00.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:00 smithi181 ceph-mon[101970]: failed to encode map e892 with expected crc 2024-03-29T05:09:00.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:00 smithi181 ceph-mon[101970]: failed to encode map e892 with expected crc 2024-03-29T05:09:00.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:00 smithi181 ceph-mon[101970]: failed to encode map e892 with expected crc 2024-03-29T05:09:00.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:00 smithi181 ceph-mon[101970]: failed to encode map e892 with expected crc 2024-03-29T05:09:00.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:00 smithi181 ceph-mon[101970]: failed to encode map e892 with expected crc 2024-03-29T05:09:00.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:00 smithi181 ceph-mon[101970]: osdmap e893: 8 total, 8 up, 8 in 2024-03-29T05:09:00.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:00 smithi181 ceph-mon[101970]: failed to encode map e893 with expected crc 2024-03-29T05:09:00.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:00 smithi181 ceph-mon[101970]: failed to encode map e893 with expected crc 2024-03-29T05:09:00.726 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:00.726 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:00.743 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:09:00.743 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:00.743 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:00.749 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:09:00.749 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:00.749 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:00.755 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:09:00.755 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:00.755 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:00.761 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:00.761 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:00.761 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:00.767 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:00.767 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:09:00.767 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:09:00.773 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,176445827~44178509,245550311~20244233,268435456~15049697,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:09:00.773 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:09:00.774 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [607349~5934701,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,143026722~4018474,147439418~4104761,176445827~44178509,246836530~4250329,251771759~14022785,269267167~10803647,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:09:00.778 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [607349~5934701,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25751882,104857600~4221929,109482434~25675982,143026722~4018474,147439418~4104761,176445827~44178509,245550311~5536548,251771759~14022785,268435456~11635358,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:09:01.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:01 smithi187 ceph-mon[93536]: failed to encode map e893 with expected crc 2024-03-29T05:09:01.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:01 smithi187 ceph-mon[93536]: failed to encode map e893 with expected crc 2024-03-29T05:09:01.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:01 smithi187 ceph-mon[93536]: failed to encode map e893 with expected crc 2024-03-29T05:09:01.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:01 smithi187 ceph-mon[93536]: pgmap v1068: 105 pgs: 105 active+clean; 818 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 117 KiB/s rd, 6.2 KiB/s wr, 1.80k op/s 2024-03-29T05:09:01.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:01 smithi187 ceph-mon[93536]: failed to encode map e894 with expected crc 2024-03-29T05:09:01.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:01 smithi187 ceph-mon[93536]: osdmap e894: 8 total, 8 up, 8 in 2024-03-29T05:09:01.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:01 smithi187 ceph-mon[93536]: failed to encode map e894 with expected crc 2024-03-29T05:09:01.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:01 smithi187 ceph-mon[93536]: failed to encode map e894 with expected crc 2024-03-29T05:09:01.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:01 smithi181 ceph-mon[106466]: failed to encode map e893 with expected crc 2024-03-29T05:09:01.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:01 smithi181 ceph-mon[106466]: failed to encode map e893 with expected crc 2024-03-29T05:09:01.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:01 smithi181 ceph-mon[106466]: failed to encode map e893 with expected crc 2024-03-29T05:09:01.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:01 smithi181 ceph-mon[106466]: pgmap v1068: 105 pgs: 105 active+clean; 818 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 117 KiB/s rd, 6.2 KiB/s wr, 1.80k op/s 2024-03-29T05:09:01.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:01 smithi181 ceph-mon[106466]: failed to encode map e894 with expected crc 2024-03-29T05:09:01.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:01 smithi181 ceph-mon[106466]: osdmap e894: 8 total, 8 up, 8 in 2024-03-29T05:09:01.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:01 smithi181 ceph-mon[106466]: failed to encode map e894 with expected crc 2024-03-29T05:09:01.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:01 smithi181 ceph-mon[106466]: failed to encode map e894 with expected crc 2024-03-29T05:09:01.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:01 smithi181 ceph-mon[101970]: failed to encode map e893 with expected crc 2024-03-29T05:09:01.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:01 smithi181 ceph-mon[101970]: failed to encode map e893 with expected crc 2024-03-29T05:09:01.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:01 smithi181 ceph-mon[101970]: failed to encode map e893 with expected crc 2024-03-29T05:09:01.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:01 smithi181 ceph-mon[101970]: pgmap v1068: 105 pgs: 105 active+clean; 818 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 117 KiB/s rd, 6.2 KiB/s wr, 1.80k op/s 2024-03-29T05:09:01.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:01 smithi181 ceph-mon[101970]: failed to encode map e894 with expected crc 2024-03-29T05:09:01.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:01 smithi181 ceph-mon[101970]: osdmap e894: 8 total, 8 up, 8 in 2024-03-29T05:09:01.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:01 smithi181 ceph-mon[101970]: failed to encode map e894 with expected crc 2024-03-29T05:09:01.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:01 smithi181 ceph-mon[101970]: failed to encode map e894 with expected crc 2024-03-29T05:09:01.725 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:01.726 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:01.745 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:09:01.745 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:01.745 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:01.751 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:09:01.751 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:01.751 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:01.757 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:09:01.757 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:01.757 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:01.763 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:01.763 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:01.763 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:01.768 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:01.768 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:09:01.769 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,176445827~44178509,246836530~18958014,269267167~14217986,286060646~3118392,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:09:01.774 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,176445827~44178509,245550311~20244233,268435456~15049697,286060646~3118392,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,365857476~12524908,381700185~25474565,407807916~10971009] 2024-03-29T05:09:02.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:02 smithi187 ceph-mon[93536]: failed to encode map e894 with expected crc 2024-03-29T05:09:02.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:02 smithi187 ceph-mon[93536]: failed to encode map e894 with expected crc 2024-03-29T05:09:02.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:02 smithi187 ceph-mon[93536]: failed to encode map e894 with expected crc 2024-03-29T05:09:02.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:02 smithi187 ceph-mon[93536]: failed to encode map e894 with expected crc 2024-03-29T05:09:02.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:02 smithi187 ceph-mon[93536]: osdmap e895: 8 total, 8 up, 8 in 2024-03-29T05:09:02.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:02 smithi187 ceph-mon[93536]: failed to encode map e895 with expected crc 2024-03-29T05:09:02.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:02 smithi187 ceph-mon[93536]: failed to encode map e895 with expected crc 2024-03-29T05:09:02.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:02 smithi181 ceph-mon[101970]: failed to encode map e894 with expected crc 2024-03-29T05:09:02.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:02 smithi181 ceph-mon[101970]: failed to encode map e894 with expected crc 2024-03-29T05:09:02.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:02 smithi181 ceph-mon[101970]: failed to encode map e894 with expected crc 2024-03-29T05:09:02.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:02 smithi181 ceph-mon[101970]: failed to encode map e894 with expected crc 2024-03-29T05:09:02.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:02 smithi181 ceph-mon[101970]: osdmap e895: 8 total, 8 up, 8 in 2024-03-29T05:09:02.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:02 smithi181 ceph-mon[101970]: failed to encode map e895 with expected crc 2024-03-29T05:09:02.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:02 smithi181 ceph-mon[101970]: failed to encode map e895 with expected crc 2024-03-29T05:09:02.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:02 smithi181 ceph-mon[106466]: failed to encode map e894 with expected crc 2024-03-29T05:09:02.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:02 smithi181 ceph-mon[106466]: failed to encode map e894 with expected crc 2024-03-29T05:09:02.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:02 smithi181 ceph-mon[106466]: failed to encode map e894 with expected crc 2024-03-29T05:09:02.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:02 smithi181 ceph-mon[106466]: failed to encode map e894 with expected crc 2024-03-29T05:09:02.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:02 smithi181 ceph-mon[106466]: osdmap e895: 8 total, 8 up, 8 in 2024-03-29T05:09:02.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:02 smithi181 ceph-mon[106466]: failed to encode map e895 with expected crc 2024-03-29T05:09:02.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:02 smithi181 ceph-mon[106466]: failed to encode map e895 with expected crc 2024-03-29T05:09:02.714 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:02.714 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:02.741 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:09:02.741 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:02.742 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:02.747 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:09:02.747 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:02.748 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:02.753 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:09:02.754 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:02.754 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:02.759 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:02.759 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:02.760 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,173301115~2201551,176445827~44178509,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:02.765 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~3924913,173301115~2201551,176445827~44178509,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:03.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:03 smithi187 ceph-mon[93536]: failed to encode map e895 with expected crc 2024-03-29T05:09:03.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:03 smithi187 ceph-mon[93536]: failed to encode map e895 with expected crc 2024-03-29T05:09:03.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:03 smithi187 ceph-mon[93536]: pgmap v1071: 105 pgs: 105 active+clean; 709 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 118 KiB/s rd, 6.2 KiB/s wr, 1.33k op/s 2024-03-29T05:09:03.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:03 smithi187 ceph-mon[93536]: failed to encode map e896 with expected crc 2024-03-29T05:09:03.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:03 smithi187 ceph-mon[93536]: osdmap e896: 8 total, 8 up, 8 in 2024-03-29T05:09:03.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:03 smithi181 ceph-mon[101970]: failed to encode map e895 with expected crc 2024-03-29T05:09:03.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:03 smithi181 ceph-mon[101970]: failed to encode map e895 with expected crc 2024-03-29T05:09:03.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:03 smithi181 ceph-mon[101970]: pgmap v1071: 105 pgs: 105 active+clean; 709 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 118 KiB/s rd, 6.2 KiB/s wr, 1.33k op/s 2024-03-29T05:09:03.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:03 smithi181 ceph-mon[101970]: failed to encode map e896 with expected crc 2024-03-29T05:09:03.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:03 smithi181 ceph-mon[101970]: osdmap e896: 8 total, 8 up, 8 in 2024-03-29T05:09:03.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:03 smithi181 ceph-mon[106466]: failed to encode map e895 with expected crc 2024-03-29T05:09:03.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:03 smithi181 ceph-mon[106466]: failed to encode map e895 with expected crc 2024-03-29T05:09:03.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:03 smithi181 ceph-mon[106466]: pgmap v1071: 105 pgs: 105 active+clean; 709 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 118 KiB/s rd, 6.2 KiB/s wr, 1.33k op/s 2024-03-29T05:09:03.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:03 smithi181 ceph-mon[106466]: failed to encode map e896 with expected crc 2024-03-29T05:09:03.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:03 smithi181 ceph-mon[106466]: osdmap e896: 8 total, 8 up, 8 in 2024-03-29T05:09:03.720 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:03.721 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:03.739 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:09:03.739 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:03.739 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:03.745 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:09:03.745 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:03.745 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:03.750 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:09:03.750 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:03.750 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~13041116,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:03.756 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,39741677~14285925,57010643~11516130,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~24254251,347366098~139655,349983031~6849925,357994231~2693797,361795334~45379416,407807916~10971009] 2024-03-29T05:09:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:04 smithi187 ceph-mon[93536]: failed to encode map e896 with expected crc 2024-03-29T05:09:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:04 smithi187 ceph-mon[93536]: failed to encode map e896 with expected crc 2024-03-29T05:09:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:04 smithi187 ceph-mon[93536]: osdmap e897: 8 total, 8 up, 8 in 2024-03-29T05:09:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:04 smithi187 ceph-mon[93536]: failed to encode map e897 with expected crc 2024-03-29T05:09:04.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:04 smithi181 ceph-mon[101970]: failed to encode map e896 with expected crc 2024-03-29T05:09:04.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:04 smithi181 ceph-mon[101970]: failed to encode map e896 with expected crc 2024-03-29T05:09:04.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:04 smithi181 ceph-mon[101970]: osdmap e897: 8 total, 8 up, 8 in 2024-03-29T05:09:04.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:04 smithi181 ceph-mon[101970]: failed to encode map e897 with expected crc 2024-03-29T05:09:04.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:04 smithi181 ceph-mon[106466]: failed to encode map e896 with expected crc 2024-03-29T05:09:04.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:04 smithi181 ceph-mon[106466]: failed to encode map e896 with expected crc 2024-03-29T05:09:04.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:04 smithi181 ceph-mon[106466]: osdmap e897: 8 total, 8 up, 8 in 2024-03-29T05:09:04.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:04 smithi181 ceph-mon[106466]: failed to encode map e897 with expected crc 2024-03-29T05:09:04.721 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:04.721 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:04.741 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:09:04.741 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:04.741 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:04.747 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:09:04.747 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:04.747 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72073222~3768797,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,232982965~7450751,246836530~18958014,269267167~19911871,293351984~1296,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:04.752 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,72073222~3768797,78594274~25992633,104857600~4221929,109482434~25675982,138412032~13132147,161975827~7234472,171966464~48657872,232982965~7450751,245550311~20244233,268435456~20743582,293351984~1296,293991835~7308510,301989888~3797272,306184192~14554656,322513851~96265074] 2024-03-29T05:09:05.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:05 smithi187 ceph-mon[93536]: failed to encode map e897 with expected crc 2024-03-29T05:09:05.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:05 smithi187 ceph-mon[93536]: pgmap v1074: 105 pgs: 105 active+clean; 673 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 84 KiB/s rd, 6.0 KiB/s wr, 1.14k op/s 2024-03-29T05:09:05.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:05 smithi187 ceph-mon[93536]: osdmap e898: 8 total, 8 up, 8 in 2024-03-29T05:09:05.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:05 smithi187 ceph-mon[93536]: failed to encode map e898 with expected crc 2024-03-29T05:09:05.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:05 smithi187 ceph-mon[93536]: failed to encode map e898 with expected crc 2024-03-29T05:09:05.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:05 smithi181 ceph-mon[101970]: failed to encode map e897 with expected crc 2024-03-29T05:09:05.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:05 smithi181 ceph-mon[101970]: pgmap v1074: 105 pgs: 105 active+clean; 673 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 84 KiB/s rd, 6.0 KiB/s wr, 1.14k op/s 2024-03-29T05:09:05.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:05 smithi181 ceph-mon[101970]: osdmap e898: 8 total, 8 up, 8 in 2024-03-29T05:09:05.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:05 smithi181 ceph-mon[101970]: failed to encode map e898 with expected crc 2024-03-29T05:09:05.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:05 smithi181 ceph-mon[101970]: failed to encode map e898 with expected crc 2024-03-29T05:09:05.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:05 smithi181 ceph-mon[106466]: failed to encode map e897 with expected crc 2024-03-29T05:09:05.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:05 smithi181 ceph-mon[106466]: pgmap v1074: 105 pgs: 105 active+clean; 673 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 84 KiB/s rd, 6.0 KiB/s wr, 1.14k op/s 2024-03-29T05:09:05.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:05 smithi181 ceph-mon[106466]: osdmap e898: 8 total, 8 up, 8 in 2024-03-29T05:09:05.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:05 smithi181 ceph-mon[106466]: failed to encode map e898 with expected crc 2024-03-29T05:09:05.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:05 smithi181 ceph-mon[106466]: failed to encode map e898 with expected crc 2024-03-29T05:09:05.723 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:05.724 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:05.742 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:09:05.742 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:05.742 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~10396580,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:05.747 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~12307948,38431381~15596221,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,161975827~7234472,171966464~48657872,226492416~13941300,245550311~20244233,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:09:06.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:06 smithi187 ceph-mon[93536]: failed to encode map e898 with expected crc 2024-03-29T05:09:06.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:06 smithi187 ceph-mon[93536]: failed to encode map e898 with expected crc 2024-03-29T05:09:06.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:06 smithi187 ceph-mon[93536]: failed to encode map e898 with expected crc 2024-03-29T05:09:06.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:06 smithi187 ceph-mon[93536]: failed to encode map e898 with expected crc 2024-03-29T05:09:06.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:06 smithi187 ceph-mon[93536]: osdmap e899: 8 total, 8 up, 8 in 2024-03-29T05:09:06.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:06 smithi187 ceph-mon[93536]: failed to encode map e899 with expected crc 2024-03-29T05:09:06.385 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:06 smithi187 ceph-mon[93536]: failed to encode map e899 with expected crc 2024-03-29T05:09:06.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:06 smithi181 ceph-mon[101970]: failed to encode map e898 with expected crc 2024-03-29T05:09:06.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:06 smithi181 ceph-mon[101970]: failed to encode map e898 with expected crc 2024-03-29T05:09:06.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:06 smithi181 ceph-mon[101970]: failed to encode map e898 with expected crc 2024-03-29T05:09:06.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:06 smithi181 ceph-mon[101970]: failed to encode map e898 with expected crc 2024-03-29T05:09:06.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:06 smithi181 ceph-mon[101970]: osdmap e899: 8 total, 8 up, 8 in 2024-03-29T05:09:06.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:06 smithi181 ceph-mon[101970]: failed to encode map e899 with expected crc 2024-03-29T05:09:06.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:06 smithi181 ceph-mon[101970]: failed to encode map e899 with expected crc 2024-03-29T05:09:06.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:06 smithi181 ceph-mon[106466]: failed to encode map e898 with expected crc 2024-03-29T05:09:06.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:06 smithi181 ceph-mon[106466]: failed to encode map e898 with expected crc 2024-03-29T05:09:06.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:06 smithi181 ceph-mon[106466]: failed to encode map e898 with expected crc 2024-03-29T05:09:06.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:06 smithi181 ceph-mon[106466]: failed to encode map e898 with expected crc 2024-03-29T05:09:06.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:06 smithi181 ceph-mon[106466]: osdmap e899: 8 total, 8 up, 8 in 2024-03-29T05:09:06.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:06 smithi181 ceph-mon[106466]: failed to encode map e899 with expected crc 2024-03-29T05:09:06.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:06 smithi181 ceph-mon[106466]: failed to encode map e899 with expected crc 2024-03-29T05:09:06.724 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:06.724 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [404568~171768,607349~9909559,13801673~10588900,27077192~11354189,39741677~14285925,56829493~11697280,72033937~3808082,78594274~25992633,106697534~2381995,109482434~10542390,120464135~6280357,127176832~7981584,139874896~11669283,156036112~645685,161975827~3924913,167413531~1796768,173109571~47514765,230584791~9848925,242552251~2998060,246836530~18958014,269267167~19911871,292525779~1466056,295753178~5547167,304147414~1639746,307919109~12819739,322513851~9531078,333726986~85051939] 2024-03-29T05:09:06.741 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~10516908,12582912~11807661,25165824~28861778,54525952~14000821,71303168~4538851,78594274~25992633,104857600~30300816,138412032~13132147,155189248~1492549,161975827~7234472,171966464~48657872,226492416~39302128,268435456~20743582,289406976~11893369,301989888~3797272,306184192~112594733] 2024-03-29T05:09:07.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:07 smithi187 ceph-mon[93536]: failed to encode map e899 with expected crc 2024-03-29T05:09:07.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:07 smithi187 ceph-mon[93536]: failed to encode map e899 with expected crc 2024-03-29T05:09:07.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:07 smithi187 ceph-mon[93536]: failed to encode map e899 with expected crc 2024-03-29T05:09:07.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:07 smithi187 ceph-mon[93536]: failed to encode map e899 with expected crc 2024-03-29T05:09:07.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:07 smithi187 ceph-mon[93536]: pgmap v1077: 105 pgs: 105 active+clean; 673 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 81 KiB/s rd, 6.0 KiB/s wr, 323 op/s 2024-03-29T05:09:07.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:07 smithi187 ceph-mon[93536]: failed to encode map e900 with expected crc 2024-03-29T05:09:07.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:07 smithi187 ceph-mon[93536]: osdmap e900: 8 total, 8 up, 8 in 2024-03-29T05:09:07.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:07 smithi187 ceph-mon[93536]: failed to encode map e900 with expected crc 2024-03-29T05:09:07.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:07 smithi181 ceph-mon[106466]: failed to encode map e899 with expected crc 2024-03-29T05:09:07.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:07 smithi181 ceph-mon[106466]: failed to encode map e899 with expected crc 2024-03-29T05:09:07.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:07 smithi181 ceph-mon[106466]: failed to encode map e899 with expected crc 2024-03-29T05:09:07.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:07 smithi181 ceph-mon[106466]: failed to encode map e899 with expected crc 2024-03-29T05:09:07.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:07 smithi181 ceph-mon[106466]: pgmap v1077: 105 pgs: 105 active+clean; 673 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 81 KiB/s rd, 6.0 KiB/s wr, 323 op/s 2024-03-29T05:09:07.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:07 smithi181 ceph-mon[106466]: failed to encode map e900 with expected crc 2024-03-29T05:09:07.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:07 smithi181 ceph-mon[106466]: osdmap e900: 8 total, 8 up, 8 in 2024-03-29T05:09:07.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:07 smithi181 ceph-mon[106466]: failed to encode map e900 with expected crc 2024-03-29T05:09:07.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:07 smithi181 ceph-mon[101970]: failed to encode map e899 with expected crc 2024-03-29T05:09:07.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:07 smithi181 ceph-mon[101970]: failed to encode map e899 with expected crc 2024-03-29T05:09:07.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:07 smithi181 ceph-mon[101970]: failed to encode map e899 with expected crc 2024-03-29T05:09:07.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:07 smithi181 ceph-mon[101970]: failed to encode map e899 with expected crc 2024-03-29T05:09:07.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:07 smithi181 ceph-mon[101970]: pgmap v1077: 105 pgs: 105 active+clean; 673 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 81 KiB/s rd, 6.0 KiB/s wr, 323 op/s 2024-03-29T05:09:07.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:07 smithi181 ceph-mon[101970]: failed to encode map e900 with expected crc 2024-03-29T05:09:07.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:07 smithi181 ceph-mon[101970]: osdmap e900: 8 total, 8 up, 8 in 2024-03-29T05:09:07.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:07 smithi181 ceph-mon[101970]: failed to encode map e900 with expected crc 2024-03-29T05:09:07.733 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/0.DiffIterateStress (39989 ms) 2024-03-29T05:09:07.733 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParent 2024-03-29T05:09:07.734 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:09:08.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:08 smithi187 ceph-mon[93536]: failed to encode map e900 with expected crc 2024-03-29T05:09:08.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:08 smithi187 ceph-mon[93536]: failed to encode map e900 with expected crc 2024-03-29T05:09:08.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:08 smithi187 ceph-mon[93536]: failed to encode map e900 with expected crc 2024-03-29T05:09:08.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:08 smithi187 ceph-mon[93536]: osdmap e901: 8 total, 8 up, 8 in 2024-03-29T05:09:08.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:08 smithi187 ceph-mon[93536]: failed to encode map e901 with expected crc 2024-03-29T05:09:08.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:08 smithi181 ceph-mon[106466]: failed to encode map e900 with expected crc 2024-03-29T05:09:08.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:08 smithi181 ceph-mon[106466]: failed to encode map e900 with expected crc 2024-03-29T05:09:08.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:08 smithi181 ceph-mon[106466]: failed to encode map e900 with expected crc 2024-03-29T05:09:08.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:08 smithi181 ceph-mon[106466]: osdmap e901: 8 total, 8 up, 8 in 2024-03-29T05:09:08.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:08 smithi181 ceph-mon[106466]: failed to encode map e901 with expected crc 2024-03-29T05:09:08.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:08 smithi181 ceph-mon[101970]: failed to encode map e900 with expected crc 2024-03-29T05:09:08.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:08 smithi181 ceph-mon[101970]: failed to encode map e900 with expected crc 2024-03-29T05:09:08.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:08 smithi181 ceph-mon[101970]: failed to encode map e900 with expected crc 2024-03-29T05:09:08.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:08 smithi181 ceph-mon[101970]: osdmap e901: 8 total, 8 up, 8 in 2024-03-29T05:09:08.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:08 smithi181 ceph-mon[101970]: failed to encode map e901 with expected crc 2024-03-29T05:09:08.821 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/0.DiffIterateParent (1088 ms) 2024-03-29T05:09:08.821 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/0.DiffIterateIgnoreParent 2024-03-29T05:09:08.822 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:09:09.204 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:09:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:09:08] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.43.0" 2024-03-29T05:09:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:09 smithi181 ceph-mon[101970]: failed to encode map e901 with expected crc 2024-03-29T05:09:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:09 smithi181 ceph-mon[101970]: failed to encode map e901 with expected crc 2024-03-29T05:09:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:09 smithi181 ceph-mon[101970]: failed to encode map e901 with expected crc 2024-03-29T05:09:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:09 smithi181 ceph-mon[101970]: failed to encode map e901 with expected crc 2024-03-29T05:09:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:09 smithi181 ceph-mon[101970]: failed to encode map e901 with expected crc 2024-03-29T05:09:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:09 smithi181 ceph-mon[101970]: pgmap v1080: 105 pgs: 105 active+clean; 556 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 346 op/s 2024-03-29T05:09:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:09 smithi181 ceph-mon[101970]: osdmap e902: 8 total, 8 up, 8 in 2024-03-29T05:09:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:09 smithi181 ceph-mon[101970]: failed to encode map e902 with expected crc 2024-03-29T05:09:09.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:09 smithi181 ceph-mon[106466]: failed to encode map e901 with expected crc 2024-03-29T05:09:09.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:09 smithi181 ceph-mon[106466]: failed to encode map e901 with expected crc 2024-03-29T05:09:09.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:09 smithi181 ceph-mon[106466]: failed to encode map e901 with expected crc 2024-03-29T05:09:09.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:09 smithi181 ceph-mon[106466]: failed to encode map e901 with expected crc 2024-03-29T05:09:09.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:09 smithi181 ceph-mon[106466]: failed to encode map e901 with expected crc 2024-03-29T05:09:09.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:09 smithi181 ceph-mon[106466]: pgmap v1080: 105 pgs: 105 active+clean; 556 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 346 op/s 2024-03-29T05:09:09.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:09 smithi181 ceph-mon[106466]: osdmap e902: 8 total, 8 up, 8 in 2024-03-29T05:09:09.205 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:09 smithi181 ceph-mon[106466]: failed to encode map e902 with expected crc 2024-03-29T05:09:09.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:09 smithi187 ceph-mon[93536]: failed to encode map e901 with expected crc 2024-03-29T05:09:09.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:09 smithi187 ceph-mon[93536]: failed to encode map e901 with expected crc 2024-03-29T05:09:09.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:09 smithi187 ceph-mon[93536]: failed to encode map e901 with expected crc 2024-03-29T05:09:09.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:09 smithi187 ceph-mon[93536]: failed to encode map e901 with expected crc 2024-03-29T05:09:09.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:09 smithi187 ceph-mon[93536]: failed to encode map e901 with expected crc 2024-03-29T05:09:09.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:09 smithi187 ceph-mon[93536]: pgmap v1080: 105 pgs: 105 active+clean; 556 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 346 op/s 2024-03-29T05:09:09.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:09 smithi187 ceph-mon[93536]: osdmap e902: 8 total, 8 up, 8 in 2024-03-29T05:09:09.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:09 smithi187 ceph-mon[93536]: failed to encode map e902 with expected crc 2024-03-29T05:09:09.843 INFO:tasks.workunit.client.0.smithi181.stdout: wrote [359094~42843,3769510~6890,6843964~13136,7724017~59487,10291033~99183,10821006~35207,12484087~71685,12733495~41646,18847266~61054,20653821~68387] to clone 2024-03-29T05:09:09.844 INFO:tasks.workunit.client.0.smithi181.stdout: diff was [0~3776400,4194304~3589200,8388608~4167164,12582912~192229,16777216~3944992] 2024-03-29T05:09:09.847 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/0.DiffIterateIgnoreParent (1026 ms) 2024-03-29T05:09:09.847 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/0.DiffIterateCallbackError 2024-03-29T05:09:09.848 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:09:09.865 INFO:tasks.workunit.client.0.smithi181.stdout: wrote [1353550~4873,2160933~32365,2924067~76845,3946307~65187,12766102~3968,13037553~32001,17588481~88623,17892120~51365,19911447~75496,20726622~15429] 2024-03-29T05:09:09.874 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/0.DiffIterateCallbackError (27 ms) 2024-03-29T05:09:09.874 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParentDiscard 2024-03-29T05:09:09.874 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:09:10.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:10 smithi187 ceph-mon[93536]: failed to encode map e902 with expected crc 2024-03-29T05:09:10.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:10 smithi187 ceph-mon[93536]: failed to encode map e903 with expected crc 2024-03-29T05:09:10.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:10 smithi187 ceph-mon[93536]: osdmap e903: 8 total, 8 up, 8 in 2024-03-29T05:09:10.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:10 smithi181 ceph-mon[106466]: failed to encode map e902 with expected crc 2024-03-29T05:09:10.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:10 smithi181 ceph-mon[106466]: failed to encode map e903 with expected crc 2024-03-29T05:09:10.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:10 smithi181 ceph-mon[106466]: osdmap e903: 8 total, 8 up, 8 in 2024-03-29T05:09:10.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:10 smithi181 ceph-mon[101970]: failed to encode map e902 with expected crc 2024-03-29T05:09:10.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:10 smithi181 ceph-mon[101970]: failed to encode map e903 with expected crc 2024-03-29T05:09:10.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:10 smithi181 ceph-mon[101970]: osdmap e903: 8 total, 8 up, 8 in 2024-03-29T05:09:11.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:11 smithi187 ceph-mon[93536]: failed to encode map e903 with expected crc 2024-03-29T05:09:11.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:11 smithi187 ceph-mon[93536]: failed to encode map e903 with expected crc 2024-03-29T05:09:11.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:11 smithi187 ceph-mon[93536]: pgmap v1083: 105 pgs: 105 active+clean; 511 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 1.0 MiB/s wr, 475 op/s 2024-03-29T05:09:11.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:11 smithi187 ceph-mon[93536]: osdmap e904: 8 total, 8 up, 8 in 2024-03-29T05:09:11.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:11 smithi187 ceph-mon[93536]: failed to encode map e904 with expected crc 2024-03-29T05:09:11.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:11 smithi181 ceph-mon[101970]: failed to encode map e903 with expected crc 2024-03-29T05:09:11.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:11 smithi181 ceph-mon[101970]: failed to encode map e903 with expected crc 2024-03-29T05:09:11.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:11 smithi181 ceph-mon[101970]: pgmap v1083: 105 pgs: 105 active+clean; 511 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 1.0 MiB/s wr, 475 op/s 2024-03-29T05:09:11.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:11 smithi181 ceph-mon[101970]: osdmap e904: 8 total, 8 up, 8 in 2024-03-29T05:09:11.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:11 smithi181 ceph-mon[101970]: failed to encode map e904 with expected crc 2024-03-29T05:09:11.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:11 smithi181 ceph-mon[106466]: failed to encode map e903 with expected crc 2024-03-29T05:09:11.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:11 smithi181 ceph-mon[106466]: failed to encode map e903 with expected crc 2024-03-29T05:09:11.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:11 smithi181 ceph-mon[106466]: pgmap v1083: 105 pgs: 105 active+clean; 511 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 1.0 MiB/s wr, 475 op/s 2024-03-29T05:09:11.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:11 smithi181 ceph-mon[106466]: osdmap e904: 8 total, 8 up, 8 in 2024-03-29T05:09:11.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:11 smithi181 ceph-mon[106466]: failed to encode map e904 with expected crc 2024-03-29T05:09:11.826 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/0.DiffIterateParentDiscard (1952 ms) 2024-03-29T05:09:11.826 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnalignedSmall 2024-03-29T05:09:11.826 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:09:11.878 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnalignedSmall (51 ms) 2024-03-29T05:09:11.878 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnaligned 2024-03-29T05:09:11.878 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:09:11.950 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnaligned (73 ms) 2024-03-29T05:09:11.950 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/0.DiffIterateTryAcquireLock 2024-03-29T05:09:11.950 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:09:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:12 smithi187 ceph-mon[93536]: failed to encode map e904 with expected crc 2024-03-29T05:09:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:12 smithi187 ceph-mon[93536]: failed to encode map e904 with expected crc 2024-03-29T05:09:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:12 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:09:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:12 smithi187 ceph-mon[93536]: osdmap e905: 8 total, 8 up, 8 in 2024-03-29T05:09:12.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:12 smithi187 ceph-mon[93536]: failed to encode map e905 with expected crc 2024-03-29T05:09:12.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:12 smithi181 ceph-mon[106466]: failed to encode map e904 with expected crc 2024-03-29T05:09:12.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:12 smithi181 ceph-mon[106466]: failed to encode map e904 with expected crc 2024-03-29T05:09:12.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:12 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:09:12.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:12 smithi181 ceph-mon[106466]: osdmap e905: 8 total, 8 up, 8 in 2024-03-29T05:09:12.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:12 smithi181 ceph-mon[106466]: failed to encode map e905 with expected crc 2024-03-29T05:09:12.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:12 smithi181 ceph-mon[101970]: failed to encode map e904 with expected crc 2024-03-29T05:09:12.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:12 smithi181 ceph-mon[101970]: failed to encode map e904 with expected crc 2024-03-29T05:09:12.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:12 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:09:12.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:12 smithi181 ceph-mon[101970]: osdmap e905: 8 total, 8 up, 8 in 2024-03-29T05:09:12.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:12 smithi181 ceph-mon[101970]: failed to encode map e905 with expected crc 2024-03-29T05:09:13.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:13 smithi187 ceph-mon[93536]: failed to encode map e905 with expected crc 2024-03-29T05:09:13.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:13 smithi187 ceph-mon[93536]: failed to encode map e905 with expected crc 2024-03-29T05:09:13.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:13 smithi187 ceph-mon[93536]: pgmap v1086: 105 pgs: 105 active+clean; 519 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.4 MiB/s wr, 366 op/s 2024-03-29T05:09:13.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:13 smithi187 ceph-mon[93536]: osdmap e906: 8 total, 8 up, 8 in 2024-03-29T05:09:13.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:13 smithi181 ceph-mon[106466]: failed to encode map e905 with expected crc 2024-03-29T05:09:13.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:13 smithi181 ceph-mon[106466]: failed to encode map e905 with expected crc 2024-03-29T05:09:13.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:13 smithi181 ceph-mon[106466]: pgmap v1086: 105 pgs: 105 active+clean; 519 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.4 MiB/s wr, 366 op/s 2024-03-29T05:09:13.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:13 smithi181 ceph-mon[106466]: osdmap e906: 8 total, 8 up, 8 in 2024-03-29T05:09:13.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:13 smithi181 ceph-mon[101970]: failed to encode map e905 with expected crc 2024-03-29T05:09:13.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:13 smithi181 ceph-mon[101970]: failed to encode map e905 with expected crc 2024-03-29T05:09:13.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:13 smithi181 ceph-mon[101970]: pgmap v1086: 105 pgs: 105 active+clean; 519 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.4 MiB/s wr, 366 op/s 2024-03-29T05:09:13.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:13 smithi181 ceph-mon[101970]: osdmap e906: 8 total, 8 up, 8 in 2024-03-29T05:09:14.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:14 smithi181 ceph-mon[101970]: failed to encode map e906 with expected crc 2024-03-29T05:09:14.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:14 smithi181 ceph-mon[101970]: failed to encode map e906 with expected crc 2024-03-29T05:09:14.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:14 smithi181 ceph-mon[101970]: failed to encode map e906 with expected crc 2024-03-29T05:09:14.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:14 smithi181 ceph-mon[106466]: failed to encode map e906 with expected crc 2024-03-29T05:09:14.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:14 smithi181 ceph-mon[106466]: failed to encode map e906 with expected crc 2024-03-29T05:09:14.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:14 smithi181 ceph-mon[106466]: failed to encode map e906 with expected crc 2024-03-29T05:09:14.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:14 smithi187 ceph-mon[93536]: failed to encode map e906 with expected crc 2024-03-29T05:09:14.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:14 smithi187 ceph-mon[93536]: failed to encode map e906 with expected crc 2024-03-29T05:09:14.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:14 smithi187 ceph-mon[93536]: failed to encode map e906 with expected crc 2024-03-29T05:09:15.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:15 smithi181 ceph-mon[101970]: pgmap v1088: 105 pgs: 105 active+clean; 542 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 10 MiB/s wr, 312 op/s 2024-03-29T05:09:15.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:15 smithi181 ceph-mon[106466]: pgmap v1088: 105 pgs: 105 active+clean; 542 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 10 MiB/s wr, 312 op/s 2024-03-29T05:09:15.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:15 smithi187 ceph-mon[93536]: pgmap v1088: 105 pgs: 105 active+clean; 542 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.4 MiB/s rd, 10 MiB/s wr, 312 op/s 2024-03-29T05:09:16.985 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/0.DiffIterateTryAcquireLock (5035 ms) 2024-03-29T05:09:16.985 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStriping 2024-03-29T05:09:16.985 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T05:09:16.985 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/0.DiffIterateStriping (0 ms) 2024-03-29T05:09:17.353 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 15 tests from DiffIterateTest/0 (69209 ms total) 2024-03-29T05:09:17.353 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:09:17.353 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 15 tests from DiffIterateTest/1, where TypeParam = DiffIterateParams 2024-03-29T05:09:17.384 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/1.DiffIterate 2024-03-29T05:09:17.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:17 smithi187 ceph-mon[93536]: pgmap v1089: 105 pgs: 105 active+clean; 542 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 8.0 MiB/s wr, 238 op/s 2024-03-29T05:09:17.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:17 smithi181 ceph-mon[106466]: pgmap v1089: 105 pgs: 105 active+clean; 542 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 8.0 MiB/s wr, 238 op/s 2024-03-29T05:09:17.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:17 smithi181 ceph-mon[101970]: pgmap v1089: 105 pgs: 105 active+clean; 542 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 8.0 MiB/s wr, 238 op/s 2024-03-29T05:09:18.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:18 smithi187 ceph-mon[93536]: failed to encode map e907 with expected crc 2024-03-29T05:09:18.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:18 smithi187 ceph-mon[93536]: osdmap e907: 8 total, 8 up, 8 in 2024-03-29T05:09:18.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:18 smithi187 ceph-mon[93536]: failed to encode map e907 with expected crc 2024-03-29T05:09:18.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:18 smithi181 ceph-mon[106466]: failed to encode map e907 with expected crc 2024-03-29T05:09:18.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:18 smithi181 ceph-mon[106466]: osdmap e907: 8 total, 8 up, 8 in 2024-03-29T05:09:18.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:18 smithi181 ceph-mon[106466]: failed to encode map e907 with expected crc 2024-03-29T05:09:18.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:18 smithi181 ceph-mon[101970]: failed to encode map e907 with expected crc 2024-03-29T05:09:18.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:18 smithi181 ceph-mon[101970]: osdmap e907: 8 total, 8 up, 8 in 2024-03-29T05:09:18.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:18 smithi181 ceph-mon[101970]: failed to encode map e907 with expected crc 2024-03-29T05:09:19.203 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:09:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:09:18] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.43.0" 2024-03-29T05:09:19.375 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:09:19.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:19 smithi187 ceph-mon[93536]: failed to encode map e907 with expected crc 2024-03-29T05:09:19.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:19 smithi187 ceph-mon[93536]: pgmap v1091: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:09:19.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:19 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:09:19.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:19 smithi187 ceph-mon[93536]: failed to encode map e908 with expected crc 2024-03-29T05:09:19.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:19 smithi187 ceph-mon[93536]: failed to encode map e908 with expected crc 2024-03-29T05:09:19.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:19 smithi187 ceph-mon[93536]: failed to encode map e908 with expected crc 2024-03-29T05:09:19.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:19 smithi187 ceph-mon[93536]: osdmap e908: 8 total, 8 up, 8 in 2024-03-29T05:09:19.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:19 smithi187 ceph-mon[93536]: failed to encode map e908 with expected crc 2024-03-29T05:09:19.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:19 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/615451364' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:09:19.635 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:19 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:09:19.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[101970]: failed to encode map e907 with expected crc 2024-03-29T05:09:19.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[101970]: pgmap v1091: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:09:19.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:09:19.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[101970]: failed to encode map e908 with expected crc 2024-03-29T05:09:19.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[101970]: failed to encode map e908 with expected crc 2024-03-29T05:09:19.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[101970]: failed to encode map e908 with expected crc 2024-03-29T05:09:19.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[101970]: osdmap e908: 8 total, 8 up, 8 in 2024-03-29T05:09:19.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[101970]: failed to encode map e908 with expected crc 2024-03-29T05:09:19.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/615451364' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:09:19.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:09:19.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[106466]: failed to encode map e907 with expected crc 2024-03-29T05:09:19.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[106466]: pgmap v1091: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:09:19.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:09:19.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[106466]: failed to encode map e908 with expected crc 2024-03-29T05:09:19.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[106466]: failed to encode map e908 with expected crc 2024-03-29T05:09:19.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[106466]: failed to encode map e908 with expected crc 2024-03-29T05:09:19.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[106466]: osdmap e908: 8 total, 8 up, 8 in 2024-03-29T05:09:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[106466]: failed to encode map e908 with expected crc 2024-03-29T05:09:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/615451364' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:09:19.706 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:19 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-29T05:09:20.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:20 smithi181 ceph-mon[101970]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:09:20.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:20 smithi181 ceph-mon[101970]: failed to encode map e909 with expected crc 2024-03-29T05:09:20.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:20 smithi181 ceph-mon[101970]: osdmap e909: 8 total, 8 up, 8 in 2024-03-29T05:09:20.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:20 smithi181 ceph-mon[101970]: failed to encode map e909 with expected crc 2024-03-29T05:09:20.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:20 smithi181 ceph-mon[101970]: failed to encode map e909 with expected crc 2024-03-29T05:09:20.455 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:20 smithi181 ceph-mon[101970]: failed to encode map e909 with expected crc 2024-03-29T05:09:20.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:20 smithi181 ceph-mon[106466]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:09:20.455 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:20 smithi181 ceph-mon[106466]: failed to encode map e909 with expected crc 2024-03-29T05:09:20.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:20 smithi181 ceph-mon[106466]: osdmap e909: 8 total, 8 up, 8 in 2024-03-29T05:09:20.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:20 smithi181 ceph-mon[106466]: failed to encode map e909 with expected crc 2024-03-29T05:09:20.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:20 smithi181 ceph-mon[106466]: failed to encode map e909 with expected crc 2024-03-29T05:09:20.456 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:20 smithi181 ceph-mon[106466]: failed to encode map e909 with expected crc 2024-03-29T05:09:20.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:20 smithi187 ceph-mon[93536]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi181-86707-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-29T05:09:20.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:20 smithi187 ceph-mon[93536]: failed to encode map e909 with expected crc 2024-03-29T05:09:20.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:20 smithi187 ceph-mon[93536]: osdmap e909: 8 total, 8 up, 8 in 2024-03-29T05:09:20.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:20 smithi187 ceph-mon[93536]: failed to encode map e909 with expected crc 2024-03-29T05:09:20.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:20 smithi187 ceph-mon[93536]: failed to encode map e909 with expected crc 2024-03-29T05:09:20.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:20 smithi187 ceph-mon[93536]: failed to encode map e909 with expected crc 2024-03-29T05:09:21.404 INFO:tasks.workunit.client.0.smithi181.stdout: wrote [51063~23714,109449~7080,4448660~27173,4965794~68305,5696352~8539,8384976~84456,9513691~19203,12135332~80033,16091241~3800,17863970~66400] 2024-03-29T05:09:21.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:21 smithi187 ceph-mon[93536]: pgmap v1094: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:09:21.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:21 smithi187 ceph-mon[93536]: osdmap e910: 8 total, 8 up, 8 in 2024-03-29T05:09:21.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:21 smithi187 ceph-mon[93536]: failed to encode map e910 with expected crc 2024-03-29T05:09:21.634 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:21 smithi187 ceph-mon[93536]: failed to encode map e910 with expected crc 2024-03-29T05:09:21.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:21 smithi181 ceph-mon[106466]: pgmap v1094: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:09:21.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:21 smithi181 ceph-mon[106466]: osdmap e910: 8 total, 8 up, 8 in 2024-03-29T05:09:21.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:21 smithi181 ceph-mon[106466]: failed to encode map e910 with expected crc 2024-03-29T05:09:21.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:21 smithi181 ceph-mon[106466]: failed to encode map e910 with expected crc 2024-03-29T05:09:21.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:21 smithi181 ceph-mon[101970]: pgmap v1094: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:09:21.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:21 smithi181 ceph-mon[101970]: osdmap e910: 8 total, 8 up, 8 in 2024-03-29T05:09:21.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:21 smithi181 ceph-mon[101970]: failed to encode map e910 with expected crc 2024-03-29T05:09:21.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:21 smithi181 ceph-mon[101970]: failed to encode map e910 with expected crc 2024-03-29T05:09:22.404 INFO:tasks.workunit.client.0.smithi181.stdout: wrote [0~20971520] 2024-03-29T05:09:22.412 INFO:tasks.workunit.client.0.smithi181.stdout: diff was [0~20971520] 2024-03-29T05:09:22.415 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/1.DiffIterate (5032 ms) 2024-03-29T05:09:22.415 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministic 2024-03-29T05:09:22.416 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:09:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:22 smithi181 ceph-mon[101970]: failed to encode map e910 with expected crc 2024-03-29T05:09:22.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:22 smithi181 ceph-mon[101970]: failed to encode map e911 with expected crc 2024-03-29T05:09:22.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:22 smithi181 ceph-mon[101970]: osdmap e911: 8 total, 8 up, 8 in 2024-03-29T05:09:22.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:22 smithi181 ceph-mon[101970]: failed to encode map e911 with expected crc 2024-03-29T05:09:22.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:22 smithi181 ceph-mon[101970]: failed to encode map e911 with expected crc 2024-03-29T05:09:22.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:22.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:22 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:22 smithi181 ceph-mon[106466]: failed to encode map e910 with expected crc 2024-03-29T05:09:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:22 smithi181 ceph-mon[106466]: failed to encode map e911 with expected crc 2024-03-29T05:09:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:22 smithi181 ceph-mon[106466]: osdmap e911: 8 total, 8 up, 8 in 2024-03-29T05:09:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:22 smithi181 ceph-mon[106466]: failed to encode map e911 with expected crc 2024-03-29T05:09:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:22 smithi181 ceph-mon[106466]: failed to encode map e911 with expected crc 2024-03-29T05:09:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:22.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:22 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:22.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:22 smithi187 ceph-mon[93536]: failed to encode map e910 with expected crc 2024-03-29T05:09:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:22 smithi187 ceph-mon[93536]: failed to encode map e911 with expected crc 2024-03-29T05:09:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:22 smithi187 ceph-mon[93536]: osdmap e911: 8 total, 8 up, 8 in 2024-03-29T05:09:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:22 smithi187 ceph-mon[93536]: failed to encode map e911 with expected crc 2024-03-29T05:09:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:22 smithi187 ceph-mon[93536]: failed to encode map e911 with expected crc 2024-03-29T05:09:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:22.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:22 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:23.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[101970]: pgmap v1097: 105 pgs: 7 unknown, 98 active+clean; 13 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 91 KiB/s wr, 20 op/s 2024-03-29T05:09:23.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[101970]: failed to encode map e912 with expected crc 2024-03-29T05:09:23.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[101970]: osdmap e912: 8 total, 8 up, 8 in 2024-03-29T05:09:23.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[101970]: failed to encode map e912 with expected crc 2024-03-29T05:09:23.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[101970]: failed to encode map e912 with expected crc 2024-03-29T05:09:23.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:23.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:23.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:09:23.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:09:23.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[106466]: pgmap v1097: 105 pgs: 7 unknown, 98 active+clean; 13 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 91 KiB/s wr, 20 op/s 2024-03-29T05:09:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[106466]: failed to encode map e912 with expected crc 2024-03-29T05:09:23.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[106466]: osdmap e912: 8 total, 8 up, 8 in 2024-03-29T05:09:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[106466]: failed to encode map e912 with expected crc 2024-03-29T05:09:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[106466]: failed to encode map e912 with expected crc 2024-03-29T05:09:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:09:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:09:23.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:23 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:23.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:23 smithi187 ceph-mon[93536]: pgmap v1097: 105 pgs: 7 unknown, 98 active+clean; 13 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 91 KiB/s wr, 20 op/s 2024-03-29T05:09:23.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:23 smithi187 ceph-mon[93536]: failed to encode map e912 with expected crc 2024-03-29T05:09:23.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:23 smithi187 ceph-mon[93536]: osdmap e912: 8 total, 8 up, 8 in 2024-03-29T05:09:23.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:23 smithi187 ceph-mon[93536]: failed to encode map e912 with expected crc 2024-03-29T05:09:23.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:23 smithi187 ceph-mon[93536]: failed to encode map e912 with expected crc 2024-03-29T05:09:23.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:23 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:23.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:23 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:23.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:23 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:09:23.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:23 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:09:23.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:23 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:24.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:24 smithi181 ceph-mon[101970]: pgmap v1099: 105 pgs: 7 unknown, 98 active+clean; 13 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 97 KiB/s wr, 22 op/s 2024-03-29T05:09:24.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:24 smithi181 ceph-mon[101970]: failed to encode map e913 with expected crc 2024-03-29T05:09:24.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:24 smithi181 ceph-mon[101970]: osdmap e913: 8 total, 8 up, 8 in 2024-03-29T05:09:24.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:24 smithi181 ceph-mon[101970]: failed to encode map e913 with expected crc 2024-03-29T05:09:24.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:24 smithi181 ceph-mon[101970]: failed to encode map e913 with expected crc 2024-03-29T05:09:24.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:24 smithi181 ceph-mon[101970]: failed to encode map e913 with expected crc 2024-03-29T05:09:24.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:24 smithi181 ceph-mon[106466]: pgmap v1099: 105 pgs: 7 unknown, 98 active+clean; 13 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 97 KiB/s wr, 22 op/s 2024-03-29T05:09:24.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:24 smithi181 ceph-mon[106466]: failed to encode map e913 with expected crc 2024-03-29T05:09:24.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:24 smithi181 ceph-mon[106466]: osdmap e913: 8 total, 8 up, 8 in 2024-03-29T05:09:24.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:24 smithi181 ceph-mon[106466]: failed to encode map e913 with expected crc 2024-03-29T05:09:24.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:24 smithi181 ceph-mon[106466]: failed to encode map e913 with expected crc 2024-03-29T05:09:24.705 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:24 smithi181 ceph-mon[106466]: failed to encode map e913 with expected crc 2024-03-29T05:09:24.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:24 smithi187 ceph-mon[93536]: pgmap v1099: 105 pgs: 7 unknown, 98 active+clean; 13 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 97 KiB/s wr, 22 op/s 2024-03-29T05:09:24.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:24 smithi187 ceph-mon[93536]: failed to encode map e913 with expected crc 2024-03-29T05:09:24.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:24 smithi187 ceph-mon[93536]: osdmap e913: 8 total, 8 up, 8 in 2024-03-29T05:09:24.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:24 smithi187 ceph-mon[93536]: failed to encode map e913 with expected crc 2024-03-29T05:09:24.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:24 smithi187 ceph-mon[93536]: failed to encode map e913 with expected crc 2024-03-29T05:09:24.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:24 smithi187 ceph-mon[93536]: failed to encode map e913 with expected crc 2024-03-29T05:09:25.440 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministic (3024 ms) 2024-03-29T05:09:25.440 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministicPP 2024-03-29T05:09:25.440 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:09:25.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:25 smithi181 ceph-mon[101970]: failed to encode map e914 with expected crc 2024-03-29T05:09:25.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:25 smithi181 ceph-mon[101970]: osdmap e914: 8 total, 8 up, 8 in 2024-03-29T05:09:25.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:25 smithi181 ceph-mon[101970]: failed to encode map e914 with expected crc 2024-03-29T05:09:25.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:25 smithi181 ceph-mon[101970]: failed to encode map e914 with expected crc 2024-03-29T05:09:25.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:25 smithi181 ceph-mon[106466]: failed to encode map e914 with expected crc 2024-03-29T05:09:25.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:25 smithi181 ceph-mon[106466]: osdmap e914: 8 total, 8 up, 8 in 2024-03-29T05:09:25.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:25 smithi181 ceph-mon[106466]: failed to encode map e914 with expected crc 2024-03-29T05:09:25.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:25 smithi181 ceph-mon[106466]: failed to encode map e914 with expected crc 2024-03-29T05:09:25.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:25 smithi187 ceph-mon[93536]: failed to encode map e914 with expected crc 2024-03-29T05:09:25.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:25 smithi187 ceph-mon[93536]: osdmap e914: 8 total, 8 up, 8 in 2024-03-29T05:09:25.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:25 smithi187 ceph-mon[93536]: failed to encode map e914 with expected crc 2024-03-29T05:09:25.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:25 smithi187 ceph-mon[93536]: failed to encode map e914 with expected crc 2024-03-29T05:09:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:26 smithi181 ceph-mon[106466]: pgmap v1102: 105 pgs: 105 active+clean; 16 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 129 KiB/s wr, 37 op/s 2024-03-29T05:09:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:26 smithi181 ceph-mon[106466]: failed to encode map e915 with expected crc 2024-03-29T05:09:26.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:26 smithi181 ceph-mon[106466]: osdmap e915: 8 total, 8 up, 8 in 2024-03-29T05:09:26.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:26 smithi181 ceph-mon[106466]: failed to encode map e915 with expected crc 2024-03-29T05:09:26.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:26 smithi181 ceph-mon[106466]: failed to encode map e915 with expected crc 2024-03-29T05:09:26.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:26 smithi181 ceph-mon[106466]: failed to encode map e915 with expected crc 2024-03-29T05:09:26.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:09:26.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:26 smithi181 ceph-mon[101970]: pgmap v1102: 105 pgs: 105 active+clean; 16 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 129 KiB/s wr, 37 op/s 2024-03-29T05:09:26.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:26 smithi181 ceph-mon[101970]: failed to encode map e915 with expected crc 2024-03-29T05:09:26.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:26 smithi181 ceph-mon[101970]: osdmap e915: 8 total, 8 up, 8 in 2024-03-29T05:09:26.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:26 smithi181 ceph-mon[101970]: failed to encode map e915 with expected crc 2024-03-29T05:09:26.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:26 smithi181 ceph-mon[101970]: failed to encode map e915 with expected crc 2024-03-29T05:09:26.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:26 smithi181 ceph-mon[101970]: failed to encode map e915 with expected crc 2024-03-29T05:09:26.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:09:26.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:26 smithi187 ceph-mon[93536]: pgmap v1102: 105 pgs: 105 active+clean; 16 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 129 KiB/s wr, 37 op/s 2024-03-29T05:09:26.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:26 smithi187 ceph-mon[93536]: failed to encode map e915 with expected crc 2024-03-29T05:09:26.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:26 smithi187 ceph-mon[93536]: osdmap e915: 8 total, 8 up, 8 in 2024-03-29T05:09:26.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:26 smithi187 ceph-mon[93536]: failed to encode map e915 with expected crc 2024-03-29T05:09:26.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:26 smithi187 ceph-mon[93536]: failed to encode map e915 with expected crc 2024-03-29T05:09:26.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:26 smithi187 ceph-mon[93536]: failed to encode map e915 with expected crc 2024-03-29T05:09:26.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:09:27.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:27 smithi181 ceph-mon[106466]: failed to encode map e916 with expected crc 2024-03-29T05:09:27.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:27 smithi181 ceph-mon[106466]: osdmap e916: 8 total, 8 up, 8 in 2024-03-29T05:09:27.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:27 smithi181 ceph-mon[106466]: failed to encode map e916 with expected crc 2024-03-29T05:09:27.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:27 smithi181 ceph-mon[106466]: failed to encode map e916 with expected crc 2024-03-29T05:09:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:27 smithi181 ceph-mon[101970]: failed to encode map e916 with expected crc 2024-03-29T05:09:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:27 smithi181 ceph-mon[101970]: osdmap e916: 8 total, 8 up, 8 in 2024-03-29T05:09:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:27 smithi181 ceph-mon[101970]: failed to encode map e916 with expected crc 2024-03-29T05:09:27.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:27 smithi181 ceph-mon[101970]: failed to encode map e916 with expected crc 2024-03-29T05:09:27.747 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministicPP (2307 ms) 2024-03-29T05:09:27.747 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDiscard 2024-03-29T05:09:27.747 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:09:27.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:27 smithi187 ceph-mon[93536]: failed to encode map e916 with expected crc 2024-03-29T05:09:27.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:27 smithi187 ceph-mon[93536]: osdmap e916: 8 total, 8 up, 8 in 2024-03-29T05:09:27.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:27 smithi187 ceph-mon[93536]: failed to encode map e916 with expected crc 2024-03-29T05:09:27.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:27 smithi187 ceph-mon[93536]: failed to encode map e916 with expected crc 2024-03-29T05:09:28.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:28 smithi181 ceph-mon[106466]: pgmap v1105: 105 pgs: 105 active+clean; 16 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 104 KiB/s wr, 39 op/s 2024-03-29T05:09:28.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:28 smithi181 ceph-mon[106466]: osdmap e917: 8 total, 8 up, 8 in 2024-03-29T05:09:28.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:28 smithi181 ceph-mon[106466]: failed to encode map e917 with expected crc 2024-03-29T05:09:28.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:28 smithi181 ceph-mon[106466]: failed to encode map e917 with expected crc 2024-03-29T05:09:28.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:28 smithi181 ceph-mon[106466]: failed to encode map e917 with expected crc 2024-03-29T05:09:28.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:28 smithi181 ceph-mon[106466]: osdmap e918: 8 total, 8 up, 8 in 2024-03-29T05:09:28.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:28 smithi181 ceph-mon[106466]: failed to encode map e918 with expected crc 2024-03-29T05:09:28.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:28 smithi181 ceph-mon[101970]: pgmap v1105: 105 pgs: 105 active+clean; 16 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 104 KiB/s wr, 39 op/s 2024-03-29T05:09:28.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:28 smithi181 ceph-mon[101970]: osdmap e917: 8 total, 8 up, 8 in 2024-03-29T05:09:28.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:28 smithi181 ceph-mon[101970]: failed to encode map e917 with expected crc 2024-03-29T05:09:28.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:28 smithi181 ceph-mon[101970]: failed to encode map e917 with expected crc 2024-03-29T05:09:28.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:28 smithi181 ceph-mon[101970]: failed to encode map e917 with expected crc 2024-03-29T05:09:28.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:28 smithi181 ceph-mon[101970]: osdmap e918: 8 total, 8 up, 8 in 2024-03-29T05:09:28.705 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:28 smithi181 ceph-mon[101970]: failed to encode map e918 with expected crc 2024-03-29T05:09:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:28 smithi187 ceph-mon[93536]: pgmap v1105: 105 pgs: 105 active+clean; 16 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 104 KiB/s wr, 39 op/s 2024-03-29T05:09:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:28 smithi187 ceph-mon[93536]: osdmap e917: 8 total, 8 up, 8 in 2024-03-29T05:09:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:28 smithi187 ceph-mon[93536]: failed to encode map e917 with expected crc 2024-03-29T05:09:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:28 smithi187 ceph-mon[93536]: failed to encode map e917 with expected crc 2024-03-29T05:09:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:28 smithi187 ceph-mon[93536]: failed to encode map e917 with expected crc 2024-03-29T05:09:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:28 smithi187 ceph-mon[93536]: osdmap e918: 8 total, 8 up, 8 in 2024-03-29T05:09:28.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:28 smithi187 ceph-mon[93536]: failed to encode map e918 with expected crc 2024-03-29T05:09:29.420 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:09:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:09:28] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2024-03-29T05:09:29.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:29 smithi181 ceph-mon[101970]: failed to encode map e918 with expected crc 2024-03-29T05:09:29.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:29 smithi181 ceph-mon[101970]: failed to encode map e918 with expected crc 2024-03-29T05:09:29.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:29 smithi181 ceph-mon[101970]: osdmap e919: 8 total, 8 up, 8 in 2024-03-29T05:09:29.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:29 smithi181 ceph-mon[101970]: failed to encode map e919 with expected crc 2024-03-29T05:09:29.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:29 smithi181 ceph-mon[106466]: failed to encode map e918 with expected crc 2024-03-29T05:09:29.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:29 smithi181 ceph-mon[106466]: failed to encode map e918 with expected crc 2024-03-29T05:09:29.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:29 smithi181 ceph-mon[106466]: osdmap e919: 8 total, 8 up, 8 in 2024-03-29T05:09:29.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:29 smithi181 ceph-mon[106466]: failed to encode map e919 with expected crc 2024-03-29T05:09:29.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:29 smithi187 ceph-mon[93536]: failed to encode map e918 with expected crc 2024-03-29T05:09:29.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:29 smithi187 ceph-mon[93536]: failed to encode map e918 with expected crc 2024-03-29T05:09:29.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:29 smithi187 ceph-mon[93536]: osdmap e919: 8 total, 8 up, 8 in 2024-03-29T05:09:29.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:29 smithi187 ceph-mon[93536]: failed to encode map e919 with expected crc 2024-03-29T05:09:30.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:30 smithi181 ceph-mon[101970]: failed to encode map e919 with expected crc 2024-03-29T05:09:30.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:30 smithi181 ceph-mon[101970]: pgmap v1109: 105 pgs: 105 active+clean; 17 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 99 KiB/s wr, 106 op/s 2024-03-29T05:09:30.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:30 smithi181 ceph-mon[101970]: osdmap e920: 8 total, 8 up, 8 in 2024-03-29T05:09:30.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:30 smithi181 ceph-mon[101970]: failed to encode map e920 with expected crc 2024-03-29T05:09:30.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:30 smithi181 ceph-mon[106466]: failed to encode map e919 with expected crc 2024-03-29T05:09:30.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:30 smithi181 ceph-mon[106466]: pgmap v1109: 105 pgs: 105 active+clean; 17 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 99 KiB/s wr, 106 op/s 2024-03-29T05:09:30.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:30 smithi181 ceph-mon[106466]: osdmap e920: 8 total, 8 up, 8 in 2024-03-29T05:09:30.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:30 smithi181 ceph-mon[106466]: failed to encode map e920 with expected crc 2024-03-29T05:09:30.760 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/1.DiffIterateDiscard (3014 ms) 2024-03-29T05:09:30.761 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/1.DiffIterateTruncate 2024-03-29T05:09:30.761 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:09:30.785 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:30 smithi187 ceph-mon[93536]: failed to encode map e919 with expected crc 2024-03-29T05:09:30.785 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:30 smithi187 ceph-mon[93536]: pgmap v1109: 105 pgs: 105 active+clean; 17 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 99 KiB/s wr, 106 op/s 2024-03-29T05:09:30.785 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:30 smithi187 ceph-mon[93536]: osdmap e920: 8 total, 8 up, 8 in 2024-03-29T05:09:30.785 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:30 smithi187 ceph-mon[93536]: failed to encode map e920 with expected crc 2024-03-29T05:09:31.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:31 smithi181 ceph-mon[106466]: failed to encode map e920 with expected crc 2024-03-29T05:09:31.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:31 smithi181 ceph-mon[106466]: failed to encode map e921 with expected crc 2024-03-29T05:09:31.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:31 smithi181 ceph-mon[106466]: osdmap e921: 8 total, 8 up, 8 in 2024-03-29T05:09:31.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:31 smithi181 ceph-mon[101970]: failed to encode map e920 with expected crc 2024-03-29T05:09:31.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:31 smithi181 ceph-mon[101970]: failed to encode map e921 with expected crc 2024-03-29T05:09:31.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:31 smithi181 ceph-mon[101970]: osdmap e921: 8 total, 8 up, 8 in 2024-03-29T05:09:31.801 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:31 smithi187 ceph-mon[93536]: failed to encode map e920 with expected crc 2024-03-29T05:09:31.801 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:31 smithi187 ceph-mon[93536]: failed to encode map e921 with expected crc 2024-03-29T05:09:31.801 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:31 smithi187 ceph-mon[93536]: osdmap e921: 8 total, 8 up, 8 in 2024-03-29T05:09:32.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:32 smithi181 ceph-mon[106466]: failed to encode map e921 with expected crc 2024-03-29T05:09:32.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:32 smithi181 ceph-mon[106466]: pgmap v1112: 105 pgs: 105 active+clean; 17 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 3.0 KiB/s wr, 43 op/s 2024-03-29T05:09:32.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:32 smithi181 ceph-mon[106466]: osdmap e922: 8 total, 8 up, 8 in 2024-03-29T05:09:32.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:32 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:32.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:32 smithi181 ceph-mon[101970]: failed to encode map e921 with expected crc 2024-03-29T05:09:32.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:32 smithi181 ceph-mon[101970]: pgmap v1112: 105 pgs: 105 active+clean; 17 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 3.0 KiB/s wr, 43 op/s 2024-03-29T05:09:32.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:32 smithi181 ceph-mon[101970]: osdmap e922: 8 total, 8 up, 8 in 2024-03-29T05:09:32.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:32 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:32.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:32 smithi187 ceph-mon[93536]: failed to encode map e921 with expected crc 2024-03-29T05:09:32.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:32 smithi187 ceph-mon[93536]: pgmap v1112: 105 pgs: 105 active+clean; 17 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 3.0 KiB/s wr, 43 op/s 2024-03-29T05:09:32.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:32 smithi187 ceph-mon[93536]: osdmap e922: 8 total, 8 up, 8 in 2024-03-29T05:09:32.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:32 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:09:33.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:33 smithi181 ceph-mon[101970]: failed to encode map e922 with expected crc 2024-03-29T05:09:33.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:33 smithi181 ceph-mon[101970]: failed to encode map e922 with expected crc 2024-03-29T05:09:33.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:33 smithi181 ceph-mon[101970]: osdmap e923: 8 total, 8 up, 8 in 2024-03-29T05:09:33.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:33 smithi181 ceph-mon[101970]: failed to encode map e923 with expected crc 2024-03-29T05:09:33.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:33 smithi181 ceph-mon[106466]: failed to encode map e922 with expected crc 2024-03-29T05:09:33.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:33 smithi181 ceph-mon[106466]: failed to encode map e922 with expected crc 2024-03-29T05:09:33.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:33 smithi181 ceph-mon[106466]: osdmap e923: 8 total, 8 up, 8 in 2024-03-29T05:09:33.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:33 smithi181 ceph-mon[106466]: failed to encode map e923 with expected crc 2024-03-29T05:09:33.803 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/1.DiffIterateTruncate (3043 ms) 2024-03-29T05:09:33.804 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/1.DiffIterateWriteAndTruncate 2024-03-29T05:09:33.804 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:09:33.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:33 smithi187 ceph-mon[93536]: failed to encode map e922 with expected crc 2024-03-29T05:09:33.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:33 smithi187 ceph-mon[93536]: failed to encode map e922 with expected crc 2024-03-29T05:09:33.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:33 smithi187 ceph-mon[93536]: osdmap e923: 8 total, 8 up, 8 in 2024-03-29T05:09:33.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:33 smithi187 ceph-mon[93536]: failed to encode map e923 with expected crc 2024-03-29T05:09:34.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:34 smithi181 ceph-mon[101970]: failed to encode map e923 with expected crc 2024-03-29T05:09:34.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:34 smithi181 ceph-mon[101970]: pgmap v1115: 105 pgs: 105 active+clean; 17 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 13 KiB/s wr, 124 op/s 2024-03-29T05:09:34.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:34 smithi181 ceph-mon[101970]: osdmap e924: 8 total, 8 up, 8 in 2024-03-29T05:09:34.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:34 smithi181 ceph-mon[106466]: failed to encode map e923 with expected crc 2024-03-29T05:09:34.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:34 smithi181 ceph-mon[106466]: pgmap v1115: 105 pgs: 105 active+clean; 17 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 13 KiB/s wr, 124 op/s 2024-03-29T05:09:34.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:34 smithi181 ceph-mon[106466]: osdmap e924: 8 total, 8 up, 8 in 2024-03-29T05:09:34.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:34 smithi187 ceph-mon[93536]: failed to encode map e923 with expected crc 2024-03-29T05:09:34.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:34 smithi187 ceph-mon[93536]: pgmap v1115: 105 pgs: 105 active+clean; 17 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 13 KiB/s wr, 124 op/s 2024-03-29T05:09:34.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:34 smithi187 ceph-mon[93536]: osdmap e924: 8 total, 8 up, 8 in 2024-03-29T05:09:35.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:35 smithi181 ceph-mon[101970]: failed to encode map e924 with expected crc 2024-03-29T05:09:35.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:35 smithi181 ceph-mon[101970]: failed to encode map e924 with expected crc 2024-03-29T05:09:35.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:35 smithi181 ceph-mon[101970]: osdmap e925: 8 total, 8 up, 8 in 2024-03-29T05:09:35.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:35 smithi181 ceph-mon[101970]: failed to encode map e925 with expected crc 2024-03-29T05:09:35.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:35 smithi181 ceph-mon[106466]: failed to encode map e924 with expected crc 2024-03-29T05:09:35.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:35 smithi181 ceph-mon[106466]: failed to encode map e924 with expected crc 2024-03-29T05:09:35.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:35 smithi181 ceph-mon[106466]: osdmap e925: 8 total, 8 up, 8 in 2024-03-29T05:09:35.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:35 smithi181 ceph-mon[106466]: failed to encode map e925 with expected crc 2024-03-29T05:09:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:35 smithi187 ceph-mon[93536]: failed to encode map e924 with expected crc 2024-03-29T05:09:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:35 smithi187 ceph-mon[93536]: failed to encode map e924 with expected crc 2024-03-29T05:09:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:35 smithi187 ceph-mon[93536]: osdmap e925: 8 total, 8 up, 8 in 2024-03-29T05:09:35.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:35 smithi187 ceph-mon[93536]: failed to encode map e925 with expected crc 2024-03-29T05:09:36.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:36 smithi181 ceph-mon[106466]: failed to encode map e925 with expected crc 2024-03-29T05:09:36.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:36 smithi181 ceph-mon[106466]: failed to encode map e925 with expected crc 2024-03-29T05:09:36.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:36 smithi181 ceph-mon[106466]: pgmap v1118: 105 pgs: 105 active+clean; 17 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 13 KiB/s wr, 127 op/s 2024-03-29T05:09:36.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:36 smithi181 ceph-mon[106466]: osdmap e926: 8 total, 8 up, 8 in 2024-03-29T05:09:36.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:36 smithi181 ceph-mon[101970]: failed to encode map e925 with expected crc 2024-03-29T05:09:36.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:36 smithi181 ceph-mon[101970]: failed to encode map e925 with expected crc 2024-03-29T05:09:36.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:36 smithi181 ceph-mon[101970]: pgmap v1118: 105 pgs: 105 active+clean; 17 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 13 KiB/s wr, 127 op/s 2024-03-29T05:09:36.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:36 smithi181 ceph-mon[101970]: osdmap e926: 8 total, 8 up, 8 in 2024-03-29T05:09:36.820 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/1.DiffIterateWriteAndTruncate (3016 ms) 2024-03-29T05:09:36.820 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStress 2024-03-29T05:09:36.820 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:09:36.875 INFO:tasks.workunit.client.0.smithi181.stdout: i=0 exists [7828638~767784,51096566~2641338,102988063~4913915,139978866~3241532,161437947~7155638,174341513~3555540,261725139~3833238,396863445~6724497] wrote [7828638~767784,51096566~2641338,102988063~4913915,139978866~3241532,161437947~7155638,174341513~3555540,261725139~3833238,396863445~6724497] 2024-03-29T05:09:36.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:36 smithi187 ceph-mon[93536]: failed to encode map e925 with expected crc 2024-03-29T05:09:36.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:36 smithi187 ceph-mon[93536]: failed to encode map e925 with expected crc 2024-03-29T05:09:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:36 smithi187 ceph-mon[93536]: pgmap v1118: 105 pgs: 105 active+clean; 17 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 13 KiB/s wr, 127 op/s 2024-03-29T05:09:36.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:36 smithi187 ceph-mon[93536]: osdmap e926: 8 total, 8 up, 8 in 2024-03-29T05:09:37.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:37 smithi181 ceph-mon[101970]: failed to encode map e926 with expected crc 2024-03-29T05:09:37.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:37 smithi181 ceph-mon[101970]: failed to encode map e926 with expected crc 2024-03-29T05:09:37.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:37 smithi181 ceph-mon[101970]: osdmap e927: 8 total, 8 up, 8 in 2024-03-29T05:09:37.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:37 smithi181 ceph-mon[106466]: failed to encode map e926 with expected crc 2024-03-29T05:09:37.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:37 smithi181 ceph-mon[106466]: failed to encode map e926 with expected crc 2024-03-29T05:09:37.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:37 smithi181 ceph-mon[106466]: osdmap e927: 8 total, 8 up, 8 in 2024-03-29T05:09:37.854 INFO:tasks.workunit.client.0.smithi181.stdout: i=1 exists [7828638~767784,10151555~6473039,29303166~7144388,37675654~2580240,51096566~2641338,102988063~4913915,108152896~6434938,139978866~3241532,150393282~5144736,161437947~7155638,174341513~5157870,220415124~707323,261725139~3833238,335497357~1536930,384157604~5365868,396863445~6724497,407127652~6596318] wrote [10151555~6473039,29303166~7144388,37675654~2580240,108152896~6434938,150393282~5144736,174566082~4933301,220415124~707323,335497357~1536930,384157604~5365868,407127652~6596318] 2024-03-29T05:09:37.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:37 smithi187 ceph-mon[93536]: failed to encode map e926 with expected crc 2024-03-29T05:09:37.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:37 smithi187 ceph-mon[93536]: failed to encode map e926 with expected crc 2024-03-29T05:09:37.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:37 smithi187 ceph-mon[93536]: osdmap e927: 8 total, 8 up, 8 in 2024-03-29T05:09:38.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:38 smithi181 ceph-mon[101970]: failed to encode map e927 with expected crc 2024-03-29T05:09:38.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:38 smithi181 ceph-mon[101970]: failed to encode map e927 with expected crc 2024-03-29T05:09:38.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:38 smithi181 ceph-mon[101970]: pgmap v1121: 105 pgs: 105 active+clean; 18 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 258 KiB/s wr, 36 op/s 2024-03-29T05:09:38.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:38 smithi181 ceph-mon[101970]: osdmap e928: 8 total, 8 up, 8 in 2024-03-29T05:09:38.704 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:38 smithi181 ceph-mon[101970]: failed to encode map e928 with expected crc 2024-03-29T05:09:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:38 smithi181 ceph-mon[106466]: failed to encode map e927 with expected crc 2024-03-29T05:09:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:38 smithi181 ceph-mon[106466]: failed to encode map e927 with expected crc 2024-03-29T05:09:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:38 smithi181 ceph-mon[106466]: pgmap v1121: 105 pgs: 105 active+clean; 18 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 258 KiB/s wr, 36 op/s 2024-03-29T05:09:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:38 smithi181 ceph-mon[106466]: osdmap e928: 8 total, 8 up, 8 in 2024-03-29T05:09:38.704 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:38 smithi181 ceph-mon[106466]: failed to encode map e928 with expected crc 2024-03-29T05:09:38.853 INFO:tasks.workunit.client.0.smithi181.stdout: i=2 exists [7828638~767784,10151555~6473039,29303166~7144388,37675654~2580240,51096566~2641338,102988063~4913915,108152896~6434938,127849190~1352326,139978866~3241532,150393282~5144736,159454422~9139163,174341513~5157870,218167760~3575289,255864354~9694023,281691423~6186466,313965567~4710022,335497357~1536930,340134405~6065032,384157604~5365868,396863445~6724497,407127652~8968989] wrote [15646218~355959,127849190~1352326,159454422~5358353,218167760~3575289,255864354~7483405,281691423~6186466,313965567~4710022,340134405~6065032,384699580~4739659,408134609~7962032] 2024-03-29T05:09:38.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:38 smithi187 ceph-mon[93536]: failed to encode map e927 with expected crc 2024-03-29T05:09:38.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:38 smithi187 ceph-mon[93536]: failed to encode map e927 with expected crc 2024-03-29T05:09:38.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:38 smithi187 ceph-mon[93536]: pgmap v1121: 105 pgs: 105 active+clean; 18 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 258 KiB/s wr, 36 op/s 2024-03-29T05:09:38.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:38 smithi187 ceph-mon[93536]: osdmap e928: 8 total, 8 up, 8 in 2024-03-29T05:09:38.884 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:38 smithi187 ceph-mon[93536]: failed to encode map e928 with expected crc 2024-03-29T05:09:39.203 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:09:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:09:38] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2024-03-29T05:09:39.464 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:39 smithi181 ceph-mon[101970]: failed to encode map e928 with expected crc 2024-03-29T05:09:39.464 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:39 smithi181 ceph-mon[101970]: failed to encode map e929 with expected crc 2024-03-29T05:09:39.464 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:39 smithi181 ceph-mon[101970]: osdmap e929: 8 total, 8 up, 8 in 2024-03-29T05:09:39.464 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:39 smithi181 ceph-mon[101970]: failed to encode map e929 with expected crc 2024-03-29T05:09:39.464 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:39 smithi181 ceph-mon[106466]: failed to encode map e928 with expected crc 2024-03-29T05:09:39.464 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:39 smithi181 ceph-mon[106466]: failed to encode map e929 with expected crc 2024-03-29T05:09:39.464 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:39 smithi181 ceph-mon[106466]: osdmap e929: 8 total, 8 up, 8 in 2024-03-29T05:09:39.465 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:39 smithi181 ceph-mon[106466]: failed to encode map e929 with expected crc 2024-03-29T05:09:39.871 INFO:tasks.workunit.client.0.smithi181.stdout: i=3 exists [7828638~767784,10151555~6473039,29303166~7144388,37675654~2580240,44597616~3893497,51096566~2641338,55971667~1864840,100926078~13661756,127849190~1352326,139978866~3241532,150393282~5144736,159454422~9139163,174341513~5157870,181559213~2872473,211851177~9891872,226260896~7423903,244622959~7528268,255864354~9694023,281691423~6186466,313965567~4710022,335497357~1536930,340134405~6065032,384157604~5365868,396863445~6724497,407127652~8968989] wrote [11659577~3720534,44597616~3893497,55971667~1864840,100926078~7951797,181559213~2872473,211851177~7538988,226260896~7423903,244622959~7528268] 2024-03-29T05:09:39.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:39 smithi187 ceph-mon[93536]: failed to encode map e928 with expected crc 2024-03-29T05:09:39.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:39 smithi187 ceph-mon[93536]: failed to encode map e929 with expected crc 2024-03-29T05:09:39.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:39 smithi187 ceph-mon[93536]: osdmap e929: 8 total, 8 up, 8 in 2024-03-29T05:09:39.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:39 smithi187 ceph-mon[93536]: failed to encode map e929 with expected crc 2024-03-29T05:09:40.881 INFO:tasks.workunit.client.0.smithi181.stdout: i=4 exists [7828638~767784,10151555~6473039,29303166~7144388,37675654~2580240,44597616~3893497,51096566~2641338,55971667~1864840,64384004~4947298,97525877~17061957,127849190~1352326,139978866~3241532,150393282~5144736,157668131~1165397,159454422~9139163,174341513~5157870,181559213~2872473,211851177~9891872,226260896~7423903,244622959~7528268,255864354~13036780,281691423~6186466,293220777~6507182,303960066~6989309,313965567~4710022,335497357~1536930,340134405~6065032,348595335~6691558,384157604~5365868,396863445~6724497,407127652~8968989] wrote [52209368~1484508,64384004~4947298,97525877~4801183,157668131~1165397,228768657~4424104,262923942~5977192,293220777~6507182,303960066~6989309,348595335~6691558] 2024-03-29T05:09:41.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:40 smithi187 ceph-mon[93536]: pgmap v1124: 105 pgs: 105 active+clean; 69 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 8.8 MiB/s wr, 189 op/s 2024-03-29T05:09:41.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:40 smithi187 ceph-mon[93536]: failed to encode map e930 with expected crc 2024-03-29T05:09:41.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:40 smithi187 ceph-mon[93536]: osdmap e930: 8 total, 8 up, 8 in 2024-03-29T05:09:41.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:40 smithi181 ceph-mon[101970]: pgmap v1124: 105 pgs: 105 active+clean; 69 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 8.8 MiB/s wr, 189 op/s 2024-03-29T05:09:41.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:40 smithi181 ceph-mon[101970]: failed to encode map e930 with expected crc 2024-03-29T05:09:41.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:40 smithi181 ceph-mon[101970]: osdmap e930: 8 total, 8 up, 8 in 2024-03-29T05:09:41.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:40 smithi181 ceph-mon[106466]: pgmap v1124: 105 pgs: 105 active+clean; 69 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 8.8 MiB/s wr, 189 op/s 2024-03-29T05:09:41.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:40 smithi181 ceph-mon[106466]: failed to encode map e930 with expected crc 2024-03-29T05:09:41.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:40 smithi181 ceph-mon[106466]: osdmap e930: 8 total, 8 up, 8 in 2024-03-29T05:09:41.865 INFO:tasks.workunit.client.0.smithi181.stdout: i=5 exists [7828638~767784,10151555~6473039,29303166~7144388,37675654~2580240,44597616~4938590,51096566~2641338,55971667~1864840,64384004~4947298,97525877~17061957,117568682~4816570,127849190~1352326,129558852~4117182,139978866~3241532,150393282~5144736,157668131~1165397,159454422~9139163,174341513~5157870,181559213~2872473,189873860~1150980,211851177~9891872,226260896~7423903,243782761~8368466,255864354~13036780,281691423~6186466,293220777~6507182,303960066~6989309,313965567~8512577,335497357~1536930,340134405~6065032,348595335~6691558,384157604~5365868,393703032~2298349,396863445~6724497,407127652~8968989] wrote [47340714~2195492,103576879~7141162,117568682~4816570,129558852~4117182,189873860~1150980,215672590~551942,243782761~5434578,316645646~5832498,393703032~2298349,400906326~1458795] 2024-03-29T05:09:42.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:41 smithi187 ceph-mon[93536]: failed to encode map e931 with expected crc 2024-03-29T05:09:42.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:41 smithi187 ceph-mon[93536]: osdmap e931: 8 total, 8 up, 8 in 2024-03-29T05:09:42.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:41 smithi187 ceph-mon[93536]: failed to encode map e931 with expected crc 2024-03-29T05:09:42.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:41 smithi187 ceph-mon[93536]: failed to encode map e931 with expected crc 2024-03-29T05:09:42.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:09:42.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:41 smithi181 ceph-mon[101970]: failed to encode map e931 with expected crc 2024-03-29T05:09:42.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:41 smithi181 ceph-mon[101970]: osdmap e931: 8 total, 8 up, 8 in 2024-03-29T05:09:42.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:41 smithi181 ceph-mon[101970]: failed to encode map e931 with expected crc 2024-03-29T05:09:42.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:41 smithi181 ceph-mon[101970]: failed to encode map e931 with expected crc 2024-03-29T05:09:42.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:09:42.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:41 smithi181 ceph-mon[106466]: failed to encode map e931 with expected crc 2024-03-29T05:09:42.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:41 smithi181 ceph-mon[106466]: osdmap e931: 8 total, 8 up, 8 in 2024-03-29T05:09:42.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:41 smithi181 ceph-mon[106466]: failed to encode map e931 with expected crc 2024-03-29T05:09:42.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:41 smithi181 ceph-mon[106466]: failed to encode map e931 with expected crc 2024-03-29T05:09:42.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:09:42.877 INFO:tasks.workunit.client.0.smithi181.stdout: i=6 exists [7828638~767784,10151555~6473039,26423008~334171,29303166~7144388,37675654~2580240,44597616~4938590,51096566~2641338,55971667~1864840,64384004~4947298,97525877~17061957,117568682~4816570,127849190~5826844,139978866~3241532,148386376~896599,150393282~18200303,174341513~5157870,181559213~2872473,189873860~1150980,211851177~9891872,226260896~7423903,243782761~8368466,255864354~13036780,277525452~2217033,281691423~6186466,289052690~10675269,303960066~6989309,313965567~10783620,335497357~1536930,340134405~6065032,348595335~6691558,362486451~7367810,384157604~5365868,393703032~2298349,396863445~6724497,407127652~8968989] wrote [26423008~334171,128064089~4727692,148386376~896599,152342085~8483361,277525452~2217033,289052690~7180172,320925090~3824097,362486451~7367810,410929629~1838545] 2024-03-29T05:09:43.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:42 smithi187 ceph-mon[93536]: pgmap v1127: 105 pgs: 105 active+clean; 69 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 94 KiB/s rd, 8.5 MiB/s wr, 155 op/s 2024-03-29T05:09:43.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:42 smithi187 ceph-mon[93536]: osdmap e932: 8 total, 8 up, 8 in 2024-03-29T05:09:43.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:42 smithi187 ceph-mon[93536]: failed to encode map e932 with expected crc 2024-03-29T05:09:43.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:42 smithi187 ceph-mon[93536]: failed to encode map e932 with expected crc 2024-03-29T05:09:43.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:42 smithi181 ceph-mon[106466]: pgmap v1127: 105 pgs: 105 active+clean; 69 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 94 KiB/s rd, 8.5 MiB/s wr, 155 op/s 2024-03-29T05:09:43.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:42 smithi181 ceph-mon[106466]: osdmap e932: 8 total, 8 up, 8 in 2024-03-29T05:09:43.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:42 smithi181 ceph-mon[106466]: failed to encode map e932 with expected crc 2024-03-29T05:09:43.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:42 smithi181 ceph-mon[106466]: failed to encode map e932 with expected crc 2024-03-29T05:09:43.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:42 smithi181 ceph-mon[101970]: pgmap v1127: 105 pgs: 105 active+clean; 69 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 94 KiB/s rd, 8.5 MiB/s wr, 155 op/s 2024-03-29T05:09:43.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:42 smithi181 ceph-mon[101970]: osdmap e932: 8 total, 8 up, 8 in 2024-03-29T05:09:43.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:42 smithi181 ceph-mon[101970]: failed to encode map e932 with expected crc 2024-03-29T05:09:43.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:42 smithi181 ceph-mon[101970]: failed to encode map e932 with expected crc 2024-03-29T05:09:43.904 INFO:tasks.workunit.client.0.smithi181.stdout: i=7 exists [326783~3730134,7828638~767784,10151555~6473039,26423008~334171,29303166~10952728,44597616~5664704,51096566~2641338,55971667~1864840,58948699~678269,64384004~4947298,97525877~17061957,117568682~4816570,127849190~5826844,139978866~3241532,148386376~896599,150393282~18200303,174341513~5157870,181559213~2872473,189873860~1150980,211851177~9891872,226260896~9176448,243782761~8368466,255588337~13312797,277525452~2217033,281691423~6186466,289052690~10675269,303960066~9836335,313965567~10783620,335497357~1536930,340134405~6065032,348595335~8996359,362486451~7367810,384157604~5365868,393703032~2298349,396863445~6724497,407127652~8968989] wrote [326783~3730134,33500477~5508219,47391830~2870490,58948699~678269,152631239~7067684,227532597~7904747,255588337~1700556,306080616~283868,309829917~3966484,351818760~5772934] 2024-03-29T05:09:44.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:43 smithi187 ceph-mon[93536]: failed to encode map e933 with expected crc 2024-03-29T05:09:44.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:43 smithi187 ceph-mon[93536]: osdmap e933: 8 total, 8 up, 8 in 2024-03-29T05:09:44.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:43 smithi187 ceph-mon[93536]: failed to encode map e933 with expected crc 2024-03-29T05:09:44.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:43 smithi181 ceph-mon[101970]: failed to encode map e933 with expected crc 2024-03-29T05:09:44.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:43 smithi181 ceph-mon[101970]: osdmap e933: 8 total, 8 up, 8 in 2024-03-29T05:09:44.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:43 smithi181 ceph-mon[101970]: failed to encode map e933 with expected crc 2024-03-29T05:09:44.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:43 smithi181 ceph-mon[106466]: failed to encode map e933 with expected crc 2024-03-29T05:09:44.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:43 smithi181 ceph-mon[106466]: osdmap e933: 8 total, 8 up, 8 in 2024-03-29T05:09:44.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:43 smithi181 ceph-mon[106466]: failed to encode map e933 with expected crc 2024-03-29T05:09:44.898 INFO:tasks.workunit.client.0.smithi181.stdout: i=8 exists [326783~3730134,7828638~767784,10151555~6473039,26423008~334171,29303166~10952728,40874550~1140437,44597616~5664704,51096566~2641338,55971667~1864840,58948699~678269,64384004~4947298,82331691~6470823,97525877~17061957,117568682~4816570,127849190~6459392,139978866~3241532,147551051~21042534,174341513~5157870,181559213~2872473,189873860~1150980,211851177~9891872,226260896~9176448,243782761~8368466,255588337~13312797,277525452~2217033,281691423~6186466,289052690~10675269,301308298~1105786,303960066~9836335,313965567~10783620,335497357~1536930,340134405~6065032,348595335~8996359,362486451~7367810,372258000~928319,384157604~5365868,393703032~2298349,396863445~6724497,407127652~8968989] wrote [40874550~1140437,82331691~6470823,131434298~2874284,147551051~6417817,265503637~1686398,301308298~1105786,351750920~5677553,372258000~928319] 2024-03-29T05:09:45.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:44 smithi187 ceph-mon[93536]: pgmap v1130: 105 pgs: 105 active+clean; 267 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 120 KiB/s rd, 41 MiB/s wr, 212 op/s 2024-03-29T05:09:45.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:44 smithi187 ceph-mon[93536]: osdmap e934: 8 total, 8 up, 8 in 2024-03-29T05:09:45.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:44 smithi187 ceph-mon[93536]: failed to encode map e934 with expected crc 2024-03-29T05:09:45.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:44 smithi187 ceph-mon[93536]: failed to encode map e934 with expected crc 2024-03-29T05:09:45.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:44 smithi181 ceph-mon[106466]: pgmap v1130: 105 pgs: 105 active+clean; 267 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 120 KiB/s rd, 41 MiB/s wr, 212 op/s 2024-03-29T05:09:45.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:44 smithi181 ceph-mon[106466]: osdmap e934: 8 total, 8 up, 8 in 2024-03-29T05:09:45.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:44 smithi181 ceph-mon[106466]: failed to encode map e934 with expected crc 2024-03-29T05:09:45.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:44 smithi181 ceph-mon[106466]: failed to encode map e934 with expected crc 2024-03-29T05:09:45.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:44 smithi181 ceph-mon[101970]: pgmap v1130: 105 pgs: 105 active+clean; 267 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 120 KiB/s rd, 41 MiB/s wr, 212 op/s 2024-03-29T05:09:45.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:44 smithi181 ceph-mon[101970]: osdmap e934: 8 total, 8 up, 8 in 2024-03-29T05:09:45.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:44 smithi181 ceph-mon[101970]: failed to encode map e934 with expected crc 2024-03-29T05:09:45.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:44 smithi181 ceph-mon[101970]: failed to encode map e934 with expected crc 2024-03-29T05:09:45.988 INFO:tasks.workunit.client.0.smithi181.stdout: i=9 exists [326783~3730134,7828638~767784,9772891~295759,10151555~11258997,26423008~334171,29303166~10952728,40874550~1140437,44597616~5664704,51096566~2641338,54851071~6902407,64384004~4947298,81663146~7427183,97525877~17061957,117568682~4816570,127849190~11937596,139978866~3241532,147551051~21042534,174341513~5157870,181559213~9933857,211851177~9891872,226260896~9176448,243782761~8368466,255588337~13312797,277525452~2217033,281691423~6186466,289052690~10675269,301308298~1105786,303960066~9836335,313965567~10783620,335497357~1536930,340134405~6065032,348595335~8996359,362486451~7367810,372258000~928319,384157604~5365868,393703032~2298349,396863445~8163634,407127652~8968989] wrote [9772891~295759,15921782~5488770,54851071~6902407,81663146~7427183,104313676~6700594,131768799~8017987,183306593~8186477,399052777~5974302,410815360~3938902] 2024-03-29T05:09:46.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:45 smithi187 ceph-mon[93536]: failed to encode map e935 with expected crc 2024-03-29T05:09:46.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:45 smithi187 ceph-mon[93536]: osdmap e935: 8 total, 8 up, 8 in 2024-03-29T05:09:46.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:45 smithi187 ceph-mon[93536]: failed to encode map e935 with expected crc 2024-03-29T05:09:46.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:45 smithi187 ceph-mon[93536]: failed to encode map e935 with expected crc 2024-03-29T05:09:46.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:45 smithi181 ceph-mon[101970]: failed to encode map e935 with expected crc 2024-03-29T05:09:46.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:45 smithi181 ceph-mon[101970]: osdmap e935: 8 total, 8 up, 8 in 2024-03-29T05:09:46.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:45 smithi181 ceph-mon[101970]: failed to encode map e935 with expected crc 2024-03-29T05:09:46.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:45 smithi181 ceph-mon[101970]: failed to encode map e935 with expected crc 2024-03-29T05:09:46.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:45 smithi181 ceph-mon[106466]: failed to encode map e935 with expected crc 2024-03-29T05:09:46.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:45 smithi181 ceph-mon[106466]: osdmap e935: 8 total, 8 up, 8 in 2024-03-29T05:09:46.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:45 smithi181 ceph-mon[106466]: failed to encode map e935 with expected crc 2024-03-29T05:09:46.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:45 smithi181 ceph-mon[106466]: failed to encode map e935 with expected crc 2024-03-29T05:09:46.918 INFO:tasks.workunit.client.0.smithi181.stdout: i=10 exists [326783~3730134,7828638~767784,9772891~295759,10151555~11258997,26423008~334171,29303166~10952728,40874550~1140437,44597616~5664704,51096566~2641338,54851071~6902407,64384004~4947298,81663146~7427183,97525877~17061957,117568682~4816570,127849190~11937596,139978866~3241532,147551051~21042534,174341513~5157870,180401164~11091906,211851177~9891872,226260896~9176448,243782761~9433641,254593804~14307330,277525452~2217033,281691423~6186466,289052690~10675269,300177086~13619315,313965567~15453091,330975215~6059072,337495136~8704301,348595335~8996359,362486451~7367810,372258000~928319,381599947~251027,384157604~5365868,393703032~2298349,396863445~8163634,407127652~8968989] wrote [180401164~5873821,247734872~444026,251982623~1233779,254593804~4750363,300177086~4437721,307483393~3432482,323169610~6249048,330975215~5876236,337495136~4985476,381599947~251027] 2024-03-29T05:09:47.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:46 smithi187 ceph-mon[93536]: pgmap v1133: 105 pgs: 105 active+clean; 315 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 51 MiB/s wr, 283 op/s 2024-03-29T05:09:47.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:46 smithi187 ceph-mon[93536]: failed to encode map e936 with expected crc 2024-03-29T05:09:47.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:46 smithi187 ceph-mon[93536]: osdmap e936: 8 total, 8 up, 8 in 2024-03-29T05:09:47.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:46 smithi187 ceph-mon[93536]: failed to encode map e936 with expected crc 2024-03-29T05:09:47.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:46 smithi181 ceph-mon[106466]: pgmap v1133: 105 pgs: 105 active+clean; 315 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 51 MiB/s wr, 283 op/s 2024-03-29T05:09:47.345 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:46 smithi181 ceph-mon[106466]: failed to encode map e936 with expected crc 2024-03-29T05:09:47.346 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:46 smithi181 ceph-mon[106466]: osdmap e936: 8 total, 8 up, 8 in 2024-03-29T05:09:47.346 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:46 smithi181 ceph-mon[106466]: failed to encode map e936 with expected crc 2024-03-29T05:09:47.346 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:46 smithi181 ceph-mon[101970]: pgmap v1133: 105 pgs: 105 active+clean; 315 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 51 MiB/s wr, 283 op/s 2024-03-29T05:09:47.346 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:46 smithi181 ceph-mon[101970]: failed to encode map e936 with expected crc 2024-03-29T05:09:47.346 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:46 smithi181 ceph-mon[101970]: osdmap e936: 8 total, 8 up, 8 in 2024-03-29T05:09:47.346 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:46 smithi181 ceph-mon[101970]: failed to encode map e936 with expected crc 2024-03-29T05:09:47.934 INFO:tasks.workunit.client.0.smithi181.stdout: i=11 exists [326783~3730134,7828638~767784,9772891~295759,10151555~11912430,26423008~334171,29303166~10952728,40874550~1140437,44597616~10188650,54851071~6902407,64384004~4947298,75094364~6549653,81663146~7427183,97525877~17061957,117568682~4816570,122396993~17389793,139978866~3241532,147551051~21042534,174341513~5157870,180401164~12543716,211851177~9891872,226260896~9176448,243782761~9433641,254593804~14307330,277525452~2217033,281691423~6186466,289052690~10675269,300177086~13619315,313965567~15453091,330975215~15224222,348595335~8996359,362486451~7621114,372258000~928319,381599947~251027,384157604~5365868,393703032~2298349,396863445~8163634,407127652~8968989] wrote [17721681~4342304,48696879~6089387,75094364~6549653,122396993~7112274,156891862~4488966,188209851~4735029,216571470~4175344,333006237~9887892,366820488~3287077] 2024-03-29T05:09:48.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:47 smithi187 ceph-mon[93536]: failed to encode map e937 with expected crc 2024-03-29T05:09:48.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:47 smithi187 ceph-mon[93536]: osdmap e937: 8 total, 8 up, 8 in 2024-03-29T05:09:48.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:47 smithi187 ceph-mon[93536]: failed to encode map e937 with expected crc 2024-03-29T05:09:48.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:47 smithi181 ceph-mon[101970]: failed to encode map e937 with expected crc 2024-03-29T05:09:48.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:47 smithi181 ceph-mon[101970]: osdmap e937: 8 total, 8 up, 8 in 2024-03-29T05:09:48.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:47 smithi181 ceph-mon[101970]: failed to encode map e937 with expected crc 2024-03-29T05:09:48.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:47 smithi181 ceph-mon[106466]: failed to encode map e937 with expected crc 2024-03-29T05:09:48.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:47 smithi181 ceph-mon[106466]: osdmap e937: 8 total, 8 up, 8 in 2024-03-29T05:09:48.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:47 smithi181 ceph-mon[106466]: failed to encode map e937 with expected crc 2024-03-29T05:09:48.933 INFO:tasks.workunit.client.0.smithi181.stdout: i=12 exists [326783~3730134,7828638~767784,9772891~295759,10151555~11912430,26423008~334171,29303166~10952728,40874550~1140437,44597616~10188650,54851071~6902407,64384004~4947298,75074084~14016245,97525877~17061957,117568682~4816570,122396993~17389793,139978866~3241532,147551051~21042534,174341513~5157870,180401164~12543716,211851177~9891872,226260896~13687838,243782761~9433641,254593804~14307330,277525452~2217033,281691423~6186466,289052690~10675269,300177086~13619315,313965567~15453091,330975215~15224222,348595335~13026692,362486451~7621114,371235486~3193829,381599947~251027,383544586~5978886,393703032~2298349,396863445~19233196] wrote [75074084~2980669,81456421~5960568,103984031~5355293,148379095~8102252,185585254~5065599,234091710~5857024,356960447~4661580,371235486~3193829,383544586~2818684,402335673~8079859] 2024-03-29T05:09:49.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:48 smithi187 ceph-mon[93536]: pgmap v1136: 105 pgs: 105 active+clean; 415 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 34 MiB/s wr, 83 op/s 2024-03-29T05:09:49.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:48 smithi187 ceph-mon[93536]: failed to encode map e938 with expected crc 2024-03-29T05:09:49.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:48 smithi187 ceph-mon[93536]: osdmap e938: 8 total, 8 up, 8 in 2024-03-29T05:09:49.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:48 smithi187 ceph-mon[93536]: failed to encode map e938 with expected crc 2024-03-29T05:09:49.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:48 smithi187 ceph-mon[93536]: failed to encode map e938 with expected crc 2024-03-29T05:09:49.203 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:09:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:09:48] "GET /metrics HTTP/1.1" 200 36536 "" "Prometheus/2.43.0" 2024-03-29T05:09:49.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:48 smithi181 ceph-mon[101970]: pgmap v1136: 105 pgs: 105 active+clean; 415 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 34 MiB/s wr, 83 op/s 2024-03-29T05:09:49.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:48 smithi181 ceph-mon[101970]: failed to encode map e938 with expected crc 2024-03-29T05:09:49.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:48 smithi181 ceph-mon[101970]: osdmap e938: 8 total, 8 up, 8 in 2024-03-29T05:09:49.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:48 smithi181 ceph-mon[101970]: failed to encode map e938 with expected crc 2024-03-29T05:09:49.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:48 smithi181 ceph-mon[101970]: failed to encode map e938 with expected crc 2024-03-29T05:09:49.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:48 smithi181 ceph-mon[106466]: pgmap v1136: 105 pgs: 105 active+clean; 415 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 34 MiB/s wr, 83 op/s 2024-03-29T05:09:49.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:48 smithi181 ceph-mon[106466]: failed to encode map e938 with expected crc 2024-03-29T05:09:49.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:48 smithi181 ceph-mon[106466]: osdmap e938: 8 total, 8 up, 8 in 2024-03-29T05:09:49.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:48 smithi181 ceph-mon[106466]: failed to encode map e938 with expected crc 2024-03-29T05:09:49.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:48 smithi181 ceph-mon[106466]: failed to encode map e938 with expected crc 2024-03-29T05:09:49.937 INFO:tasks.workunit.client.0.smithi181.stdout: i=13 exists [326783~3730134,7828638~767784,9772891~295759,10151555~14907596,26423008~334171,29303166~10952728,40874550~1140437,44597616~10188650,54851071~6902407,64384004~4947298,75074084~16749728,97525877~17061957,117568682~4816570,122396993~17389793,139978866~3241532,147551051~21042534,172726967~6772416,180401164~12543716,211851177~9891872,226260896~13687838,243782761~9433641,254593804~14684249,277525452~2217033,281691423~6186466,289052690~10675269,300177086~13619315,313965567~15453091,330975215~15224222,348595335~13026692,362486451~8489142,371235486~3193829,377601368~4862660,383544586~5978886,393703032~2298349,396863445~19233196] wrote [19251569~5807582,29898109~2152269,49139569~4712961,88296140~3527672,105380505~2152258,172726967~6143230,264292306~4985747,365111250~5864343,377601368~4862660] 2024-03-29T05:09:50.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:49 smithi187 ceph-mon[93536]: failed to encode map e939 with expected crc 2024-03-29T05:09:50.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:49 smithi187 ceph-mon[93536]: osdmap e939: 8 total, 8 up, 8 in 2024-03-29T05:09:50.134 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:49 smithi187 ceph-mon[93536]: failed to encode map e939 with expected crc 2024-03-29T05:09:50.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:49 smithi181 ceph-mon[106466]: failed to encode map e939 with expected crc 2024-03-29T05:09:50.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:49 smithi181 ceph-mon[106466]: osdmap e939: 8 total, 8 up, 8 in 2024-03-29T05:09:50.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:49 smithi181 ceph-mon[106466]: failed to encode map e939 with expected crc 2024-03-29T05:09:50.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:49 smithi181 ceph-mon[101970]: failed to encode map e939 with expected crc 2024-03-29T05:09:50.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:49 smithi181 ceph-mon[101970]: osdmap e939: 8 total, 8 up, 8 in 2024-03-29T05:09:50.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:49 smithi181 ceph-mon[101970]: failed to encode map e939 with expected crc 2024-03-29T05:09:50.928 INFO:tasks.workunit.client.0.smithi181.stdout: i=14 exists [326783~3730134,4979550~933379,7828638~767784,9772891~295759,10151555~14907596,26423008~334171,28485804~11770090,40874550~1140437,44597616~10188650,54851071~6902407,64384004~4947298,75074084~16749728,97525877~17061957,117568682~25651716,147551051~21042534,172726967~6772416,180401164~12543716,206473573~839814,211851177~9891872,226260896~13687838,243782761~9433641,254593804~14684249,277525452~2217033,281691423~6186466,289052690~10675269,300177086~13619315,313965567~15453091,330975215~15224222,347115110~1009414,348595335~13026692,362486451~8489142,371235486~3193829,377601368~4862660,383544586~5978886,393703032~2298349,396863445~19233196] wrote [4979550~933379,28485804~6239245,121792893~2864441,133971783~7725344,206473573~839814,249132717~3364519,321778480~4893246,334868383~2827299,347115110~1009414,363145998~6486900] 2024-03-29T05:09:51.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:50 smithi187 ceph-mon[93536]: pgmap v1139: 105 pgs: 105 active+clean; 523 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 201 KiB/s rd, 48 MiB/s wr, 327 op/s 2024-03-29T05:09:51.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:50 smithi187 ceph-mon[93536]: failed to encode map e940 with expected crc 2024-03-29T05:09:51.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:50 smithi187 ceph-mon[93536]: osdmap e940: 8 total, 8 up, 8 in 2024-03-29T05:09:51.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:50 smithi187 ceph-mon[93536]: failed to encode map e940 with expected crc 2024-03-29T05:09:51.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:50 smithi181 ceph-mon[101970]: pgmap v1139: 105 pgs: 105 active+clean; 523 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 201 KiB/s rd, 48 MiB/s wr, 327 op/s 2024-03-29T05:09:51.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:50 smithi181 ceph-mon[101970]: failed to encode map e940 with expected crc 2024-03-29T05:09:51.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:50 smithi181 ceph-mon[101970]: osdmap e940: 8 total, 8 up, 8 in 2024-03-29T05:09:51.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:50 smithi181 ceph-mon[101970]: failed to encode map e940 with expected crc 2024-03-29T05:09:51.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:50 smithi181 ceph-mon[106466]: pgmap v1139: 105 pgs: 105 active+clean; 523 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 201 KiB/s rd, 48 MiB/s wr, 327 op/s 2024-03-29T05:09:51.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:50 smithi181 ceph-mon[106466]: failed to encode map e940 with expected crc 2024-03-29T05:09:51.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:50 smithi181 ceph-mon[106466]: osdmap e940: 8 total, 8 up, 8 in 2024-03-29T05:09:51.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:50 smithi181 ceph-mon[106466]: failed to encode map e940 with expected crc 2024-03-29T05:09:51.961 INFO:tasks.workunit.client.0.smithi181.stdout: i=15 exists [326783~3730134,4979550~933379,7828638~767784,9772891~295759,10151555~14907596,26423008~334171,28009779~12246115,40874550~1140437,44597616~10188650,54851071~6902407,64384004~7841618,75074084~16749728,97525877~19473447,117568682~25651716,147551051~21042534,172726967~6772416,180401164~12543716,206473573~839814,211851177~12562193,226260896~13687838,243782761~9433641,254593804~14684249,271498983~2986743,277525452~2217033,281691423~6186466,289052690~10675269,300177086~13619315,313965567~15453091,330975215~15224222,347115110~1009414,348595335~13026692,362486451~8489142,371235486~3193829,377601368~4862660,383544586~5978886,393703032~2298349,396863445~19233196] wrote [28009779~761639,69162073~3063549,109075831~7923493,118193713~4309818,181564437~5852378,219686319~4727051,261509520~7238519,271498983~2986743,315276909~5701169,387991628~1167030] 2024-03-29T05:09:52.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:51 smithi187 ceph-mon[93536]: failed to encode map e941 with expected crc 2024-03-29T05:09:52.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:51 smithi187 ceph-mon[93536]: osdmap e941: 8 total, 8 up, 8 in 2024-03-29T05:09:52.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:51 smithi187 ceph-mon[93536]: failed to encode map e941 with expected crc 2024-03-29T05:09:52.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:51 smithi181 ceph-mon[101970]: failed to encode map e941 with expected crc 2024-03-29T05:09:52.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:51 smithi181 ceph-mon[101970]: osdmap e941: 8 total, 8 up, 8 in 2024-03-29T05:09:52.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:51 smithi181 ceph-mon[101970]: failed to encode map e941 with expected crc 2024-03-29T05:09:52.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:51 smithi181 ceph-mon[106466]: failed to encode map e941 with expected crc 2024-03-29T05:09:52.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:51 smithi181 ceph-mon[106466]: osdmap e941: 8 total, 8 up, 8 in 2024-03-29T05:09:52.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:51 smithi181 ceph-mon[106466]: failed to encode map e941 with expected crc 2024-03-29T05:09:52.955 INFO:tasks.workunit.client.0.smithi181.stdout: i=16 exists [326783~3730134,4979550~933379,7828638~767784,9772891~295759,10151555~14907596,26423008~334171,28009779~14050477,44597616~10188650,54851071~6902407,64384004~7841618,75074084~16749728,97525877~19473447,117568682~25651716,147551051~21042534,172726967~6772416,180401164~12543716,206473573~839814,210083411~667726,211851177~12562193,226260896~13687838,243782761~9433641,253612770~15665283,271498983~2986743,277525452~2217033,281691423~6186466,289052690~10675269,300177086~13619315,313965567~15453091,330975215~15224222,347115110~1009414,348595335~13026692,362486451~8489142,371235486~3193829,377601368~4862660,383544586~5978886,393703032~2298349,396863445~19233196] wrote [28855525~1822283,35454660~6605596,210083411~667726,253612770~11227013,308661647~2663083,333738341~39946] 2024-03-29T05:09:53.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:52 smithi187 ceph-mon[93536]: pgmap v1142: 105 pgs: 105 active+clean; 523 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 201 KiB/s rd, 23 MiB/s wr, 314 op/s 2024-03-29T05:09:53.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:52 smithi187 ceph-mon[93536]: failed to encode map e942 with expected crc 2024-03-29T05:09:53.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:52 smithi187 ceph-mon[93536]: osdmap e942: 8 total, 8 up, 8 in 2024-03-29T05:09:53.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:52 smithi187 ceph-mon[93536]: failed to encode map e942 with expected crc 2024-03-29T05:09:53.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:52 smithi181 ceph-mon[106466]: pgmap v1142: 105 pgs: 105 active+clean; 523 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 201 KiB/s rd, 23 MiB/s wr, 314 op/s 2024-03-29T05:09:53.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:52 smithi181 ceph-mon[106466]: failed to encode map e942 with expected crc 2024-03-29T05:09:53.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:52 smithi181 ceph-mon[106466]: osdmap e942: 8 total, 8 up, 8 in 2024-03-29T05:09:53.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:52 smithi181 ceph-mon[106466]: failed to encode map e942 with expected crc 2024-03-29T05:09:53.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:52 smithi181 ceph-mon[101970]: pgmap v1142: 105 pgs: 105 active+clean; 523 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 201 KiB/s rd, 23 MiB/s wr, 314 op/s 2024-03-29T05:09:53.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:52 smithi181 ceph-mon[101970]: failed to encode map e942 with expected crc 2024-03-29T05:09:53.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:52 smithi181 ceph-mon[101970]: osdmap e942: 8 total, 8 up, 8 in 2024-03-29T05:09:53.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:52 smithi181 ceph-mon[101970]: failed to encode map e942 with expected crc 2024-03-29T05:09:53.944 INFO:tasks.workunit.client.0.smithi181.stdout: i=17 exists [326783~3730134,4979550~933379,7828638~767784,9772891~295759,10151555~14907596,26423008~334171,28009779~14050477,44597616~10188650,54851071~6902407,64384004~7841618,75074084~16749728,94777083~1531080,97525877~19473447,117568682~25651716,145498620~23094965,172726967~6772416,180401164~12543716,206473573~839814,210083411~667726,211456934~12956436,226260896~13687838,243782761~9433641,253612770~15665283,271498983~2986743,277525452~2217033,281691423~6186466,289052690~10675269,300177086~13619315,313965567~15453091,330975215~15224222,347115110~1009414,348595335~13026692,362486451~8489142,371235486~3193829,377601368~4862660,383544586~5978886,393703032~2298349,396863445~19233196] wrote [94777083~1531080,141444762~264559,145498620~4821892,182692253~9906161,211456934~2794482,294039782~1214404,318521911~3954058] 2024-03-29T05:09:54.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:53 smithi181 ceph-mon[101970]: failed to encode map e943 with expected crc 2024-03-29T05:09:54.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:53 smithi181 ceph-mon[101970]: osdmap e943: 8 total, 8 up, 8 in 2024-03-29T05:09:54.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:53 smithi181 ceph-mon[106466]: failed to encode map e943 with expected crc 2024-03-29T05:09:54.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:53 smithi181 ceph-mon[106466]: osdmap e943: 8 total, 8 up, 8 in 2024-03-29T05:09:54.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:53 smithi187 ceph-mon[93536]: failed to encode map e943 with expected crc 2024-03-29T05:09:54.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:53 smithi187 ceph-mon[93536]: osdmap e943: 8 total, 8 up, 8 in 2024-03-29T05:09:54.951 INFO:tasks.workunit.client.0.smithi181.stdout: i=18 exists [326783~3730134,4979550~933379,7828638~767784,9772891~295759,10151555~14907596,26423008~334171,28009779~14050477,44597616~10188650,54851071~6902407,64384004~7841618,75074084~16749728,94777083~1531080,97525877~19473447,117568682~25651716,145498620~23094965,172726967~6772416,180401164~12543716,206473573~839814,210083411~667726,211456934~12956436,226057883~13890851,243782761~9433641,253612770~15665283,271498983~2986743,277525452~2217033,281691423~6186466,289052690~24743711,313965567~15453091,330975215~15224222,347115110~1009414,348535096~13086931,362486451~8489142,371235486~3827797,377601368~4862660,383544586~5978886,393703032~2298349,396863445~19233196] wrote [102243208~7881043,125832012~312621,131192066~347528,146724791~7741638,226057883~5859645,261797655~79725,295334175~7707954,320541395~1402938,348535096~769734,371859563~3203720] 2024-03-29T05:09:55.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:54 smithi181 ceph-mon[106466]: pgmap v1145: 105 pgs: 105 active+clean; 682 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 126 KiB/s rd, 39 MiB/s wr, 221 op/s 2024-03-29T05:09:55.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:54 smithi181 ceph-mon[106466]: failed to encode map e944 with expected crc 2024-03-29T05:09:55.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:54 smithi181 ceph-mon[106466]: osdmap e944: 8 total, 8 up, 8 in 2024-03-29T05:09:55.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:54 smithi181 ceph-mon[106466]: failed to encode map e944 with expected crc 2024-03-29T05:09:55.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:54 smithi181 ceph-mon[101970]: pgmap v1145: 105 pgs: 105 active+clean; 682 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 126 KiB/s rd, 39 MiB/s wr, 221 op/s 2024-03-29T05:09:55.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:54 smithi181 ceph-mon[101970]: failed to encode map e944 with expected crc 2024-03-29T05:09:55.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:54 smithi181 ceph-mon[101970]: osdmap e944: 8 total, 8 up, 8 in 2024-03-29T05:09:55.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:54 smithi181 ceph-mon[101970]: failed to encode map e944 with expected crc 2024-03-29T05:09:55.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:54 smithi187 ceph-mon[93536]: pgmap v1145: 105 pgs: 105 active+clean; 682 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 126 KiB/s rd, 39 MiB/s wr, 221 op/s 2024-03-29T05:09:55.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:54 smithi187 ceph-mon[93536]: failed to encode map e944 with expected crc 2024-03-29T05:09:55.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:54 smithi187 ceph-mon[93536]: osdmap e944: 8 total, 8 up, 8 in 2024-03-29T05:09:55.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:54 smithi187 ceph-mon[93536]: failed to encode map e944 with expected crc 2024-03-29T05:09:55.962 INFO:tasks.workunit.client.0.smithi181.stdout: i=19 exists [326783~3730134,4979550~933379,7828638~767784,9772891~295759,10151555~14907596,26423008~334171,28009779~14050477,44597616~10188650,54851071~6902407,64384004~7841618,75074084~16749728,94777083~1531080,97525877~19473447,117568682~25651716,145162019~23431566,172726967~6772416,180401164~12543716,199891906~1721522,206473573~839814,210083411~667726,211456934~12956436,226057883~16313732,243782761~9433641,253612770~15665283,271498983~2986743,277525452~2217033,281691423~6186466,289052690~24743711,313965567~15453091,330975215~15224222,347115110~1009414,348535096~13086931,362486451~8489142,371235486~3827797,375970049~6493979,383544586~5978886,393703032~2298349,396863445~19233196] wrote [29090271~6971530,103615474~2356665,119238509~7805725,145162019~457473,190070807~2526439,199891906~1721522,238216221~4155394,257343277~5666760,324068880~3528416,375970049~4215440] 2024-03-29T05:09:56.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:55 smithi181 ceph-mon[106466]: failed to encode map e945 with expected crc 2024-03-29T05:09:56.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:55 smithi181 ceph-mon[106466]: osdmap e945: 8 total, 8 up, 8 in 2024-03-29T05:09:56.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:55 smithi181 ceph-mon[106466]: failed to encode map e945 with expected crc 2024-03-29T05:09:56.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:55 smithi181 ceph-mon[101970]: failed to encode map e945 with expected crc 2024-03-29T05:09:56.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:55 smithi181 ceph-mon[101970]: osdmap e945: 8 total, 8 up, 8 in 2024-03-29T05:09:56.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:55 smithi181 ceph-mon[101970]: failed to encode map e945 with expected crc 2024-03-29T05:09:56.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:55 smithi187 ceph-mon[93536]: failed to encode map e945 with expected crc 2024-03-29T05:09:56.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:55 smithi187 ceph-mon[93536]: osdmap e945: 8 total, 8 up, 8 in 2024-03-29T05:09:56.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:55 smithi187 ceph-mon[93536]: failed to encode map e945 with expected crc 2024-03-29T05:09:56.924 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 1 diff [8388608~8388608,25165824~12582912,104857600~8388608,146800640~8388608,171966464~8388608,218103808~4194304,331350016~4194304,381681664~8388608,406847488~8388608] 2024-03-29T05:09:56.924 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~8388608,25165824~12582912,104857600~8388608,146800640~8388608,171966464~8388608,218103808~4194304,331350016~4194304,381681664~8388608,406847488~8388608] 2024-03-29T05:09:56.934 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~8388608,25165824~16777216,104857600~12582912,146800640~12582912,171966464~8388608,218103808~4194304,331350016~8388608,381681664~8388608,406847488~8388608] 2024-03-29T05:09:56.934 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 2 diff [8388608~8388608,25165824~12582912,104857600~8388608,125829120~4194304,146800640~8388608,159383552~8388608,171966464~8388608,218103808~4194304,255852544~8388608,281018368~8388608,310378496~8388608,331350016~4194304,339738624~8388608,381681664~8388608,406847488~12582912] 2024-03-29T05:09:56.934 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~8388608,25165824~12582912,104857600~8388608,125829120~4194304,146800640~8388608,159383552~8388608,171966464~8388608,218103808~4194304,255852544~8388608,281018368~8388608,310378496~8388608,331350016~4194304,339738624~8388608,381681664~8388608,406847488~12582912] 2024-03-29T05:09:56.938 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~8388608,25165824~16777216,104857600~12582912,125829120~4194304,146800640~20971520,171966464~8388608,218103808~4194304,255852544~8388608,281018368~8388608,310378496~8388608,331350016~16777216,381681664~8388608,406847488~12582912] 2024-03-29T05:09:56.938 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 3 diff [8388608~8388608,25165824~12582912,41943040~4194304,54525952~4194304,100663296~16777216,125829120~4194304,146800640~8388608,159383552~8388608,171966464~12582912,209715200~20971520,243269632~8388608,255852544~8388608,281018368~8388608,310378496~8388608,331350016~4194304,339738624~8388608,381681664~8388608,406847488~12582912] 2024-03-29T05:09:56.938 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~8388608,25165824~12582912,41943040~4194304,54525952~4194304,100663296~16777216,125829120~4194304,146800640~8388608,159383552~8388608,171966464~12582912,209715200~20971520,243269632~8388608,255852544~8388608,281018368~8388608,310378496~8388608,331350016~4194304,339738624~8388608,381681664~8388608,406847488~12582912] 2024-03-29T05:09:56.940 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~8388608,25165824~25165824,54525952~4194304,100663296~16777216,125829120~4194304,146800640~20971520,171966464~12582912,209715200~25165824,243269632~20971520,281018368~8388608,310378496~8388608,331350016~16777216,381681664~8388608,406847488~12582912] 2024-03-29T05:09:56.940 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 4 diff [8388608~8388608,25165824~12582912,41943040~4194304,50331648~8388608,62914560~8388608,96468992~20971520,125829120~4194304,146800640~20971520,171966464~12582912,209715200~20971520,243269632~8388608,255852544~16777216,281018368~16777216,301989888~16777216,331350016~4194304,339738624~16777216,381681664~8388608,406847488~12582912] 2024-03-29T05:09:56.940 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~8388608,25165824~12582912,41943040~4194304,50331648~8388608,62914560~8388608,96468992~20971520,125829120~4194304,146800640~20971520,171966464~12582912,209715200~20971520,243269632~8388608,255852544~16777216,281018368~16777216,301989888~16777216,331350016~4194304,339738624~16777216,381681664~8388608,406847488~12582912] 2024-03-29T05:09:56.943 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~8388608,25165824~33554432,62914560~8388608,96468992~20971520,125829120~4194304,146800640~20971520,171966464~12582912,209715200~25165824,243269632~29360128,281018368~37748736,331350016~25165824,381681664~8388608,406847488~12582912] 2024-03-29T05:09:56.943 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 5 diff [8388608~8388608,25165824~12582912,41943040~16777216,62914560~8388608,96468992~33554432,146800640~20971520,171966464~12582912,188743680~4194304,209715200~20971520,243269632~8388608,255852544~16777216,281018368~16777216,301989888~20971520,331350016~4194304,339738624~16777216,381681664~12582912,398458880~4194304,406847488~12582912] 2024-03-29T05:09:56.943 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~8388608,25165824~12582912,41943040~16777216,62914560~8388608,96468992~33554432,146800640~20971520,171966464~12582912,188743680~4194304,209715200~20971520,243269632~8388608,255852544~16777216,281018368~16777216,301989888~20971520,331350016~4194304,339738624~16777216,381681664~12582912,398458880~4194304,406847488~12582912] 2024-03-29T05:09:56.946 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~8388608,25165824~33554432,62914560~8388608,96468992~37748736,146800640~20971520,171966464~12582912,188743680~4194304,209715200~25165824,243269632~29360128,281018368~41943040,331350016~25165824,381681664~20971520,406847488~12582912] 2024-03-29T05:09:56.946 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 6 diff [8388608~8388608,25165824~12582912,41943040~16777216,62914560~8388608,96468992~37748736,146800640~16777216,171966464~12582912,188743680~4194304,209715200~20971520,243269632~8388608,255852544~16777216,276824064~20971520,301989888~20971520,331350016~4194304,339738624~16777216,360710144~8388608,381681664~12582912,398458880~4194304,406847488~12582912] 2024-03-29T05:09:56.946 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~8388608,25165824~12582912,41943040~16777216,62914560~8388608,96468992~37748736,146800640~16777216,171966464~12582912,188743680~4194304,209715200~20971520,243269632~8388608,255852544~16777216,276824064~20971520,301989888~20971520,331350016~4194304,339738624~16777216,360710144~8388608,381681664~12582912,398458880~4194304,406847488~12582912] 2024-03-29T05:09:56.950 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~8388608,25165824~33554432,62914560~8388608,96468992~37748736,146800640~20971520,171966464~12582912,188743680~4194304,209715200~25165824,243269632~29360128,276824064~50331648,331350016~25165824,360710144~12582912,381681664~20971520,406847488~12582912] 2024-03-29T05:09:56.950 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 7 diff [0~4194304,8388608~8388608,25165824~12582912,41943040~29360128,96468992~37748736,146800640~16777216,171966464~12582912,188743680~4194304,209715200~25165824,243269632~25165824,276824064~20971520,301989888~20971520,331350016~4194304,339738624~29360128,381681664~12582912,398458880~4194304,406847488~12582912] 2024-03-29T05:09:56.950 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~8388608,25165824~12582912,41943040~29360128,96468992~37748736,146800640~16777216,171966464~12582912,188743680~4194304,209715200~25165824,243269632~25165824,276824064~20971520,301989888~20971520,331350016~4194304,339738624~29360128,381681664~12582912,398458880~4194304,406847488~12582912] 2024-03-29T05:09:56.955 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~8388608,25165824~46137344,96468992~37748736,146800640~20971520,171966464~12582912,188743680~4194304,209715200~29360128,243269632~29360128,276824064~50331648,331350016~41943040,381681664~20971520,406847488~12582912] 2024-03-29T05:09:56.955 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 8 diff [0~4194304,8388608~8388608,25165824~46137344,79691776~8388608,96468992~37748736,146800640~20971520,171966464~12582912,188743680~4194304,209715200~25165824,243269632~25165824,276824064~46137344,331350016~4194304,339738624~33554432,381681664~12582912,398458880~4194304,406847488~12582912] 2024-03-29T05:09:56.955 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~8388608,25165824~46137344,79691776~8388608,96468992~37748736,146800640~20971520,171966464~12582912,188743680~4194304,209715200~25165824,243269632~25165824,276824064~46137344,331350016~4194304,339738624~33554432,381681664~12582912,398458880~4194304,406847488~12582912] 2024-03-29T05:09:56.960 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~8388608,25165824~46137344,79691776~12582912,96468992~41943040,146800640~20971520,171966464~12582912,188743680~4194304,209715200~29360128,243269632~29360128,276824064~50331648,331350016~41943040,381681664~20971520,406847488~12582912] 2024-03-29T05:09:56.960 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 9 diff [0~4194304,8388608~12582912,25165824~46137344,79691776~8388608,96468992~41943040,146800640~20971520,171966464~20971520,209715200~25165824,243269632~25165824,276824064~46137344,331350016~4194304,339738624~33554432,381681664~12582912,398458880~20971520] 2024-03-29T05:09:56.960 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~46137344,79691776~8388608,96468992~41943040,146800640~20971520,171966464~20971520,209715200~25165824,243269632~25165824,276824064~46137344,331350016~4194304,339738624~33554432,381681664~12582912,398458880~20971520] 2024-03-29T05:09:56.964 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~62914560,79691776~12582912,96468992~46137344,146800640~20971520,171966464~20971520,209715200~29360128,243269632~29360128,276824064~50331648,331350016~41943040,381681664~37748736] 2024-03-29T05:09:56.965 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 10 diff [0~4194304,8388608~12582912,25165824~46137344,79691776~8388608,96468992~41943040,146800640~20971520,171966464~20971520,209715200~25165824,243269632~25165824,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:56.965 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~46137344,79691776~8388608,96468992~41943040,146800640~20971520,171966464~20971520,209715200~25165824,243269632~25165824,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:56.970 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~62914560,79691776~12582912,96468992~46137344,146800640~20971520,171966464~20971520,209715200~29360128,243269632~29360128,276824064~96468992,377487360~41943040] 2024-03-29T05:09:56.970 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 11 diff [0~4194304,8388608~12582912,25165824~62914560,96468992~46137344,146800640~20971520,171966464~20971520,209715200~25165824,243269632~25165824,276824064~67108864,348127232~25165824,377487360~16777216,398458880~20971520] 2024-03-29T05:09:56.970 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~62914560,96468992~46137344,146800640~20971520,171966464~20971520,209715200~25165824,243269632~25165824,276824064~67108864,348127232~25165824,377487360~16777216,398458880~20971520] 2024-03-29T05:09:56.976 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~83886080,96468992~46137344,146800640~20971520,171966464~25165824,209715200~29360128,243269632~29360128,276824064~96468992,377487360~41943040] 2024-03-29T05:09:56.976 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 12 diff [0~4194304,8388608~12582912,25165824~62914560,96468992~46137344,146800640~20971520,171966464~20971520,209715200~29360128,243269632~25165824,276824064~67108864,348127232~25165824,377487360~16777216,398458880~20971520] 2024-03-29T05:09:56.976 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~62914560,96468992~46137344,146800640~20971520,171966464~20971520,209715200~29360128,243269632~25165824,276824064~67108864,348127232~25165824,377487360~16777216,398458880~20971520] 2024-03-29T05:09:56.983 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~83886080,96468992~46137344,146800640~20971520,171966464~25165824,209715200~62914560,276824064~142606336] 2024-03-29T05:09:56.983 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 13 diff [0~4194304,8388608~79691776,96468992~46137344,146800640~20971520,171966464~20971520,209715200~29360128,243269632~25165824,276824064~67108864,348127232~25165824,377487360~16777216,398458880~20971520] 2024-03-29T05:09:56.983 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~79691776,96468992~46137344,146800640~20971520,171966464~20971520,209715200~29360128,243269632~25165824,276824064~67108864,348127232~25165824,377487360~16777216,398458880~20971520] 2024-03-29T05:09:56.990 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~83886080,96468992~46137344,146800640~20971520,171966464~25165824,209715200~62914560,276824064~142606336] 2024-03-29T05:09:56.990 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 14 diff [0~88080384,96468992~46137344,146800640~20971520,171966464~20971520,205520896~33554432,243269632~25165824,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:56.990 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,96468992~46137344,146800640~20971520,171966464~20971520,205520896~33554432,243269632~25165824,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:56.997 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~92274688,96468992~46137344,146800640~20971520,171966464~25165824,205520896~67108864,276824064~142606336] 2024-03-29T05:09:56.997 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 15 diff [0~88080384,96468992~46137344,146800640~20971520,171966464~20971520,205520896~33554432,243269632~29360128,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:56.997 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,96468992~46137344,146800640~20971520,171966464~20971520,205520896~33554432,243269632~29360128,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.005 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~92274688,96468992~46137344,146800640~20971520,171966464~25165824,205520896~213909504] 2024-03-29T05:09:57.005 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 16 diff [0~88080384,96468992~46137344,146800640~20971520,171966464~20971520,205520896~33554432,243269632~29360128,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.005 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,96468992~46137344,146800640~20971520,171966464~20971520,205520896~33554432,243269632~29360128,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.013 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~92274688,96468992~46137344,146800640~20971520,171966464~25165824,205520896~213909504] 2024-03-29T05:09:57.013 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 17 diff [0~88080384,92274688~71303168,171966464~20971520,205520896~33554432,243269632~29360128,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.013 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,205520896~33554432,243269632~29360128,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.020 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~25165824,205520896~213909504] 2024-03-29T05:09:57.020 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 18 diff [0~88080384,92274688~71303168,171966464~20971520,205520896~33554432,243269632~29360128,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.020 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,205520896~33554432,243269632~29360128,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.028 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~25165824,205520896~213909504] 2024-03-29T05:09:57.028 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to 19 diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:09:57.028 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:09:57.038 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:09:57.038 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 2 diff [12582912~4194304,125829120~4194304,159383552~8388608,218103808~4194304,255852544~8388608,281018368~8388608,310378496~8388608,339738624~8388608,381681664~8388608,406847488~8388608] 2024-03-29T05:09:57.038 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [12582912~4194304,125829120~4194304,159383552~8388608,218103808~4194304,255852544~8388608,281018368~8388608,310378496~8388608,339738624~8388608,381681664~8388608,406847488~8388608] 2024-03-29T05:09:57.039 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [12582912~4194304,125829120~4194304,159383552~8388608,218103808~4194304,255852544~8388608,281018368~8388608,310378496~8388608,339738624~8388608,381681664~8388608,406847488~12582912] 2024-03-29T05:09:57.039 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 3 diff [8388608~8388608,41943040~4194304,54525952~4194304,100663296~8388608,125829120~4194304,159383552~8388608,180355072~4194304,209715200~20971520,243269632~8388608,255852544~8388608,281018368~8388608,310378496~8388608,339738624~8388608,381681664~8388608,406847488~8388608] 2024-03-29T05:09:57.039 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~8388608,41943040~4194304,54525952~4194304,100663296~8388608,125829120~4194304,159383552~8388608,180355072~4194304,209715200~20971520,243269632~8388608,255852544~8388608,281018368~8388608,310378496~8388608,339738624~8388608,381681664~8388608,406847488~8388608] 2024-03-29T05:09:57.041 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~8388608,41943040~8388608,54525952~4194304,100663296~8388608,125829120~4194304,159383552~8388608,180355072~4194304,209715200~25165824,243269632~20971520,281018368~8388608,310378496~8388608,339738624~8388608,381681664~8388608,406847488~12582912] 2024-03-29T05:09:57.041 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 4 diff [8388608~8388608,41943040~4194304,50331648~8388608,62914560~8388608,96468992~12582912,125829120~4194304,155189248~12582912,180355072~4194304,209715200~20971520,243269632~8388608,255852544~16777216,281018368~16777216,301989888~16777216,339738624~16777216,381681664~8388608,406847488~8388608] 2024-03-29T05:09:57.041 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~8388608,41943040~4194304,50331648~8388608,62914560~8388608,96468992~12582912,125829120~4194304,155189248~12582912,180355072~4194304,209715200~20971520,243269632~8388608,255852544~16777216,281018368~16777216,301989888~16777216,339738624~16777216,381681664~8388608,406847488~8388608] 2024-03-29T05:09:57.043 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~8388608,41943040~16777216,62914560~8388608,96468992~12582912,125829120~4194304,155189248~12582912,180355072~4194304,209715200~25165824,243269632~29360128,281018368~37748736,339738624~16777216,381681664~8388608,406847488~12582912] 2024-03-29T05:09:57.043 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 5 diff [8388608~8388608,41943040~16777216,62914560~8388608,96468992~16777216,117440512~12582912,155189248~12582912,180355072~4194304,188743680~4194304,209715200~20971520,243269632~8388608,255852544~16777216,281018368~16777216,301989888~20971520,339738624~16777216,381681664~12582912,398458880~4194304,406847488~8388608] 2024-03-29T05:09:57.044 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~8388608,41943040~16777216,62914560~8388608,96468992~16777216,117440512~12582912,155189248~12582912,180355072~4194304,188743680~4194304,209715200~20971520,243269632~8388608,255852544~16777216,281018368~16777216,301989888~20971520,339738624~16777216,381681664~12582912,398458880~4194304,406847488~8388608] 2024-03-29T05:09:57.046 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~8388608,41943040~16777216,62914560~8388608,96468992~16777216,117440512~16777216,155189248~12582912,180355072~4194304,188743680~4194304,209715200~25165824,243269632~29360128,281018368~41943040,339738624~16777216,381681664~20971520,406847488~12582912] 2024-03-29T05:09:57.046 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 6 diff [8388608~8388608,25165824~4194304,41943040~16777216,62914560~8388608,96468992~16777216,117440512~16777216,146800640~16777216,180355072~4194304,188743680~4194304,209715200~20971520,243269632~8388608,255852544~16777216,276824064~20971520,301989888~20971520,339738624~16777216,360710144~8388608,381681664~12582912,398458880~4194304,406847488~8388608] 2024-03-29T05:09:57.046 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~8388608,25165824~4194304,41943040~16777216,62914560~8388608,96468992~16777216,117440512~16777216,146800640~16777216,180355072~4194304,188743680~4194304,209715200~20971520,243269632~8388608,255852544~16777216,276824064~20971520,301989888~20971520,339738624~16777216,360710144~8388608,381681664~12582912,398458880~4194304,406847488~8388608] 2024-03-29T05:09:57.049 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~8388608,25165824~4194304,41943040~16777216,62914560~8388608,96468992~16777216,117440512~16777216,146800640~20971520,180355072~4194304,188743680~4194304,209715200~25165824,243269632~29360128,276824064~50331648,339738624~16777216,360710144~12582912,381681664~20971520,406847488~12582912] 2024-03-29T05:09:57.049 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 7 diff [0~4194304,8388608~8388608,25165824~12582912,41943040~29360128,96468992~16777216,117440512~16777216,146800640~16777216,180355072~4194304,188743680~4194304,209715200~25165824,243269632~25165824,276824064~20971520,301989888~20971520,339738624~29360128,381681664~12582912,398458880~4194304,406847488~8388608] 2024-03-29T05:09:57.049 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~8388608,25165824~12582912,41943040~29360128,96468992~16777216,117440512~16777216,146800640~16777216,180355072~4194304,188743680~4194304,209715200~25165824,243269632~25165824,276824064~20971520,301989888~20971520,339738624~29360128,381681664~12582912,398458880~4194304,406847488~8388608] 2024-03-29T05:09:57.052 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~8388608,25165824~46137344,96468992~16777216,117440512~16777216,146800640~20971520,180355072~4194304,188743680~4194304,209715200~29360128,243269632~29360128,276824064~50331648,339738624~33554432,381681664~20971520,406847488~12582912] 2024-03-29T05:09:57.052 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 8 diff [0~4194304,8388608~8388608,25165824~46137344,79691776~8388608,96468992~16777216,117440512~16777216,146800640~20971520,180355072~4194304,188743680~4194304,209715200~25165824,243269632~25165824,276824064~46137344,339738624~33554432,381681664~12582912,398458880~4194304,406847488~8388608] 2024-03-29T05:09:57.052 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~8388608,25165824~46137344,79691776~8388608,96468992~16777216,117440512~16777216,146800640~20971520,180355072~4194304,188743680~4194304,209715200~25165824,243269632~25165824,276824064~46137344,339738624~33554432,381681664~12582912,398458880~4194304,406847488~8388608] 2024-03-29T05:09:57.056 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~8388608,25165824~46137344,79691776~12582912,96468992~16777216,117440512~20971520,146800640~20971520,180355072~4194304,188743680~4194304,209715200~29360128,243269632~29360128,276824064~50331648,339738624~33554432,381681664~20971520,406847488~12582912] 2024-03-29T05:09:57.056 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 9 diff [0~4194304,8388608~12582912,25165824~46137344,79691776~8388608,96468992~16777216,117440512~20971520,146800640~20971520,180355072~12582912,209715200~25165824,243269632~25165824,276824064~46137344,339738624~33554432,381681664~12582912,398458880~16777216] 2024-03-29T05:09:57.056 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~46137344,79691776~8388608,96468992~16777216,117440512~20971520,146800640~20971520,180355072~12582912,209715200~25165824,243269632~25165824,276824064~46137344,339738624~33554432,381681664~12582912,398458880~16777216] 2024-03-29T05:09:57.060 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~62914560,79691776~12582912,96468992~16777216,117440512~25165824,146800640~20971520,180355072~12582912,209715200~29360128,243269632~29360128,276824064~50331648,339738624~33554432,381681664~37748736] 2024-03-29T05:09:57.060 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 10 diff [0~4194304,8388608~12582912,25165824~46137344,79691776~8388608,96468992~16777216,117440512~20971520,146800640~20971520,180355072~12582912,209715200~25165824,243269632~25165824,276824064~96468992,377487360~16777216,398458880~16777216] 2024-03-29T05:09:57.060 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~46137344,79691776~8388608,96468992~16777216,117440512~20971520,146800640~20971520,180355072~12582912,209715200~25165824,243269632~25165824,276824064~96468992,377487360~16777216,398458880~16777216] 2024-03-29T05:09:57.064 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~62914560,79691776~12582912,96468992~16777216,117440512~25165824,146800640~20971520,180355072~12582912,209715200~29360128,243269632~29360128,276824064~96468992,377487360~41943040] 2024-03-29T05:09:57.064 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 11 diff [0~4194304,8388608~12582912,25165824~62914560,96468992~16777216,117440512~25165824,146800640~20971520,180355072~12582912,209715200~25165824,243269632~25165824,276824064~67108864,348127232~25165824,377487360~16777216,398458880~16777216] 2024-03-29T05:09:57.064 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~62914560,96468992~16777216,117440512~25165824,146800640~20971520,180355072~12582912,209715200~25165824,243269632~25165824,276824064~67108864,348127232~25165824,377487360~16777216,398458880~16777216] 2024-03-29T05:09:57.069 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~83886080,96468992~16777216,117440512~25165824,146800640~20971520,180355072~16777216,209715200~29360128,243269632~29360128,276824064~96468992,377487360~41943040] 2024-03-29T05:09:57.069 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 12 diff [0~4194304,8388608~12582912,25165824~62914560,96468992~16777216,117440512~25165824,146800640~20971520,180355072~12582912,209715200~29360128,243269632~25165824,276824064~67108864,348127232~25165824,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.070 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~62914560,96468992~16777216,117440512~25165824,146800640~20971520,180355072~12582912,209715200~29360128,243269632~25165824,276824064~67108864,348127232~25165824,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.074 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~83886080,96468992~16777216,117440512~25165824,146800640~20971520,180355072~16777216,209715200~62914560,276824064~142606336] 2024-03-29T05:09:57.074 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 13 diff [0~4194304,8388608~79691776,96468992~16777216,117440512~25165824,146800640~20971520,171966464~20971520,209715200~29360128,243269632~25165824,276824064~67108864,348127232~25165824,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.074 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~79691776,96468992~16777216,117440512~25165824,146800640~20971520,171966464~20971520,209715200~29360128,243269632~25165824,276824064~67108864,348127232~25165824,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.080 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~83886080,96468992~16777216,117440512~25165824,146800640~20971520,171966464~25165824,209715200~62914560,276824064~142606336] 2024-03-29T05:09:57.080 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 14 diff [0~88080384,96468992~16777216,117440512~25165824,146800640~20971520,171966464~20971520,205520896~33554432,243269632~25165824,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.080 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,96468992~16777216,117440512~25165824,146800640~20971520,171966464~20971520,205520896~33554432,243269632~25165824,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.087 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~92274688,96468992~16777216,117440512~25165824,146800640~20971520,171966464~25165824,205520896~67108864,276824064~142606336] 2024-03-29T05:09:57.088 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 15 diff [0~88080384,96468992~46137344,146800640~20971520,171966464~20971520,205520896~33554432,243269632~29360128,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.088 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,96468992~46137344,146800640~20971520,171966464~20971520,205520896~33554432,243269632~29360128,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.095 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~92274688,96468992~46137344,146800640~20971520,171966464~25165824,205520896~213909504] 2024-03-29T05:09:57.095 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 16 diff [0~88080384,96468992~46137344,146800640~20971520,171966464~20971520,205520896~33554432,243269632~29360128,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.095 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,96468992~46137344,146800640~20971520,171966464~20971520,205520896~33554432,243269632~29360128,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.102 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~92274688,96468992~46137344,146800640~20971520,171966464~25165824,205520896~213909504] 2024-03-29T05:09:57.102 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 17 diff [0~88080384,92274688~71303168,171966464~20971520,205520896~33554432,243269632~29360128,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.102 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,205520896~33554432,243269632~29360128,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.108 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~25165824,205520896~213909504] 2024-03-29T05:09:57.108 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 18 diff [0~88080384,92274688~71303168,171966464~20971520,205520896~33554432,243269632~29360128,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.108 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,205520896~33554432,243269632~29360128,276824064~96468992,377487360~16777216,398458880~20971520] 2024-03-29T05:09:57.115 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~25165824,205520896~213909504] 2024-03-29T05:09:57.116 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to 19 diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:09:57.116 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:09:57.123 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:09:57.123 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 3 diff [8388608~4194304,41943040~4194304,54525952~4194304,100663296~8388608,180355072~4194304,209715200~8388608,222298112~8388608,243269632~8388608] 2024-03-29T05:09:57.123 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~4194304,41943040~4194304,54525952~4194304,100663296~8388608,180355072~4194304,209715200~8388608,222298112~8388608,243269632~8388608] 2024-03-29T05:09:57.125 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~8388608,41943040~8388608,54525952~4194304,100663296~8388608,180355072~4194304,209715200~25165824,243269632~12582912] 2024-03-29T05:09:57.125 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 4 diff [8388608~4194304,41943040~4194304,50331648~8388608,62914560~8388608,96468992~12582912,155189248~4194304,180355072~4194304,209715200~8388608,222298112~8388608,243269632~8388608,260046848~8388608,289406976~8388608,301989888~8388608,348127232~8388608] 2024-03-29T05:09:57.125 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~4194304,41943040~4194304,50331648~8388608,62914560~8388608,96468992~12582912,155189248~4194304,180355072~4194304,209715200~8388608,222298112~8388608,243269632~8388608,260046848~8388608,289406976~8388608,301989888~8388608,348127232~8388608] 2024-03-29T05:09:57.126 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~8388608,41943040~16777216,62914560~8388608,96468992~12582912,155189248~4194304,180355072~4194304,209715200~25165824,243269632~12582912,260046848~12582912,289406976~25165824,348127232~8388608] 2024-03-29T05:09:57.126 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 5 diff [8388608~4194304,41943040~16777216,62914560~8388608,96468992~16777216,117440512~12582912,155189248~4194304,180355072~4194304,188743680~4194304,209715200~8388608,222298112~8388608,243269632~8388608,260046848~8388608,289406976~8388608,301989888~8388608,314572800~8388608,348127232~8388608,390070272~4194304,398458880~4194304] 2024-03-29T05:09:57.126 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~4194304,41943040~16777216,62914560~8388608,96468992~16777216,117440512~12582912,155189248~4194304,180355072~4194304,188743680~4194304,209715200~8388608,222298112~8388608,243269632~8388608,260046848~8388608,289406976~8388608,301989888~8388608,314572800~8388608,348127232~8388608,390070272~4194304,398458880~4194304] 2024-03-29T05:09:57.128 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~8388608,41943040~16777216,62914560~8388608,96468992~16777216,117440512~16777216,155189248~4194304,180355072~4194304,188743680~4194304,209715200~25165824,243269632~12582912,260046848~12582912,289406976~33554432,348127232~8388608,390070272~12582912] 2024-03-29T05:09:57.128 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 6 diff [8388608~4194304,25165824~4194304,41943040~16777216,62914560~8388608,96468992~16777216,117440512~16777216,146800640~16777216,180355072~4194304,188743680~4194304,209715200~8388608,222298112~8388608,243269632~8388608,260046848~8388608,276824064~4194304,285212672~12582912,301989888~8388608,314572800~8388608,348127232~8388608,360710144~8388608,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.128 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~4194304,25165824~4194304,41943040~16777216,62914560~8388608,96468992~16777216,117440512~16777216,146800640~16777216,180355072~4194304,188743680~4194304,209715200~8388608,222298112~8388608,243269632~8388608,260046848~8388608,276824064~4194304,285212672~12582912,301989888~8388608,314572800~8388608,348127232~8388608,360710144~8388608,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.130 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~8388608,25165824~4194304,41943040~16777216,62914560~8388608,96468992~16777216,117440512~16777216,146800640~16777216,180355072~4194304,188743680~4194304,209715200~25165824,243269632~12582912,260046848~12582912,276824064~4194304,285212672~41943040,348127232~8388608,360710144~12582912,390070272~12582912,406847488~8388608] 2024-03-29T05:09:57.130 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 7 diff [0~4194304,8388608~4194304,25165824~12582912,41943040~29360128,96468992~16777216,117440512~16777216,146800640~16777216,180355072~4194304,188743680~4194304,209715200~8388608,222298112~12582912,243269632~12582912,260046848~8388608,276824064~4194304,285212672~12582912,301989888~20971520,348127232~20971520,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.131 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~4194304,25165824~12582912,41943040~29360128,96468992~16777216,117440512~16777216,146800640~16777216,180355072~4194304,188743680~4194304,209715200~8388608,222298112~12582912,243269632~12582912,260046848~8388608,276824064~4194304,285212672~12582912,301989888~20971520,348127232~20971520,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.133 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~8388608,25165824~46137344,96468992~16777216,117440512~16777216,146800640~16777216,180355072~4194304,188743680~4194304,209715200~29360128,243269632~29360128,276824064~4194304,285212672~41943040,348127232~25165824,390070272~12582912,406847488~8388608] 2024-03-29T05:09:57.133 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 8 diff [0~4194304,8388608~4194304,25165824~46137344,79691776~8388608,96468992~16777216,117440512~16777216,146800640~16777216,180355072~4194304,188743680~4194304,209715200~8388608,222298112~12582912,243269632~12582912,260046848~8388608,276824064~4194304,285212672~37748736,348127232~25165824,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.133 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~4194304,25165824~46137344,79691776~8388608,96468992~16777216,117440512~16777216,146800640~16777216,180355072~4194304,188743680~4194304,209715200~8388608,222298112~12582912,243269632~12582912,260046848~8388608,276824064~4194304,285212672~37748736,348127232~25165824,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.136 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~8388608,25165824~46137344,79691776~12582912,96468992~16777216,117440512~20971520,146800640~16777216,180355072~4194304,188743680~4194304,209715200~29360128,243269632~29360128,276824064~4194304,285212672~41943040,348127232~25165824,390070272~12582912,406847488~8388608] 2024-03-29T05:09:57.136 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 9 diff [0~4194304,8388608~12582912,25165824~46137344,79691776~8388608,96468992~16777216,117440512~20971520,146800640~16777216,180355072~12582912,209715200~8388608,222298112~12582912,243269632~12582912,260046848~8388608,276824064~4194304,285212672~37748736,348127232~25165824,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.136 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~46137344,79691776~8388608,96468992~16777216,117440512~20971520,146800640~16777216,180355072~12582912,209715200~8388608,222298112~12582912,243269632~12582912,260046848~8388608,276824064~4194304,285212672~37748736,348127232~25165824,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.139 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~62914560,79691776~12582912,96468992~16777216,117440512~25165824,146800640~16777216,180355072~12582912,209715200~29360128,243269632~29360128,276824064~4194304,285212672~41943040,348127232~25165824,390070272~25165824] 2024-03-29T05:09:57.139 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 10 diff [0~4194304,8388608~12582912,25165824~46137344,79691776~8388608,96468992~16777216,117440512~20971520,146800640~16777216,180355072~12582912,209715200~8388608,222298112~12582912,243269632~25165824,276824064~4194304,285212672~58720256,348127232~25165824,377487360~4194304,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.139 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~46137344,79691776~8388608,96468992~16777216,117440512~20971520,146800640~16777216,180355072~12582912,209715200~8388608,222298112~12582912,243269632~25165824,276824064~4194304,285212672~58720256,348127232~25165824,377487360~4194304,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.143 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~62914560,79691776~12582912,96468992~16777216,117440512~25165824,146800640~16777216,180355072~12582912,209715200~29360128,243269632~29360128,276824064~4194304,285212672~58720256,348127232~25165824,377487360~8388608,390070272~25165824] 2024-03-29T05:09:57.143 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 11 diff [0~4194304,8388608~12582912,25165824~62914560,96468992~16777216,117440512~25165824,146800640~16777216,180355072~12582912,209715200~25165824,243269632~25165824,276824064~4194304,285212672~54525952,348127232~25165824,377487360~4194304,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.143 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~62914560,96468992~16777216,117440512~25165824,146800640~16777216,180355072~12582912,209715200~25165824,243269632~25165824,276824064~4194304,285212672~54525952,348127232~25165824,377487360~4194304,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.147 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~83886080,96468992~16777216,117440512~25165824,146800640~16777216,180355072~16777216,209715200~29360128,243269632~29360128,276824064~4194304,285212672~58720256,348127232~25165824,377487360~8388608,390070272~25165824] 2024-03-29T05:09:57.147 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 12 diff [0~4194304,8388608~12582912,25165824~62914560,96468992~16777216,117440512~25165824,146800640~16777216,180355072~12582912,209715200~29360128,243269632~25165824,276824064~4194304,285212672~54525952,348127232~25165824,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.147 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~62914560,96468992~16777216,117440512~25165824,146800640~16777216,180355072~12582912,209715200~29360128,243269632~25165824,276824064~4194304,285212672~54525952,348127232~25165824,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.153 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~83886080,96468992~16777216,117440512~25165824,146800640~16777216,180355072~16777216,209715200~62914560,276824064~4194304,285212672~58720256,348127232~67108864] 2024-03-29T05:09:57.154 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 13 diff [0~4194304,8388608~79691776,96468992~16777216,117440512~25165824,146800640~16777216,171966464~20971520,209715200~29360128,243269632~25165824,276824064~4194304,285212672~54525952,348127232~25165824,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.154 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~79691776,96468992~16777216,117440512~25165824,146800640~16777216,171966464~20971520,209715200~29360128,243269632~25165824,276824064~4194304,285212672~54525952,348127232~25165824,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.157 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~83886080,96468992~16777216,117440512~25165824,146800640~16777216,171966464~25165824,209715200~62914560,276824064~4194304,285212672~58720256,348127232~67108864] 2024-03-29T05:09:57.157 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 14 diff [0~88080384,96468992~16777216,117440512~25165824,146800640~16777216,171966464~20971520,205520896~33554432,243269632~25165824,276824064~4194304,285212672~54525952,343932928~29360128,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.157 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,96468992~16777216,117440512~25165824,146800640~16777216,171966464~20971520,205520896~33554432,243269632~25165824,276824064~4194304,285212672~54525952,343932928~29360128,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.163 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~92274688,96468992~16777216,117440512~25165824,146800640~16777216,171966464~25165824,205520896~67108864,276824064~4194304,285212672~130023424] 2024-03-29T05:09:57.163 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 15 diff [0~88080384,96468992~46137344,146800640~16777216,171966464~20971520,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.163 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,96468992~46137344,146800640~16777216,171966464~20971520,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.169 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~92274688,96468992~46137344,146800640~16777216,171966464~25165824,205520896~75497472,285212672~130023424] 2024-03-29T05:09:57.169 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 16 diff [0~88080384,96468992~46137344,146800640~16777216,171966464~20971520,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.169 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,96468992~46137344,146800640~16777216,171966464~20971520,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~92274688,96468992~46137344,146800640~16777216,171966464~25165824,205520896~75497472,285212672~130023424] 2024-03-29T05:09:57.175 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 17 diff [0~88080384,92274688~67108864,171966464~20971520,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.175 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.184 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~25165824,205520896~75497472,285212672~130023424] 2024-03-29T05:09:57.184 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 18 diff [0~88080384,92274688~67108864,171966464~20971520,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.184 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.191 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~25165824,205520896~75497472,285212672~130023424] 2024-03-29T05:09:57.191 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to 19 diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:09:57.191 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:09:57.198 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:09:57.198 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 4 diff [50331648~4194304,62914560~8388608,96468992~8388608,155189248~4194304,226492416~8388608,260046848~8388608,289406976~8388608,301989888~8388608,348127232~8388608] 2024-03-29T05:09:57.198 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [50331648~4194304,62914560~8388608,96468992~8388608,155189248~4194304,226492416~8388608,260046848~8388608,289406976~8388608,301989888~8388608,348127232~8388608] 2024-03-29T05:09:57.198 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [50331648~4194304,62914560~8388608,96468992~8388608,155189248~4194304,226492416~8388608,260046848~12582912,289406976~25165824,348127232~8388608] 2024-03-29T05:09:57.198 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 5 diff [46137344~8388608,62914560~8388608,96468992~12582912,117440512~12582912,155189248~4194304,188743680~4194304,213909504~4194304,226492416~8388608,243269632~8388608,260046848~8388608,289406976~8388608,301989888~8388608,314572800~8388608,348127232~8388608,390070272~4194304,398458880~4194304] 2024-03-29T05:09:57.199 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [46137344~8388608,62914560~8388608,96468992~12582912,117440512~12582912,155189248~4194304,188743680~4194304,213909504~4194304,226492416~8388608,243269632~8388608,260046848~8388608,289406976~8388608,301989888~8388608,314572800~8388608,348127232~8388608,390070272~4194304,398458880~4194304] 2024-03-29T05:09:57.200 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [46137344~8388608,62914560~8388608,96468992~16777216,117440512~16777216,155189248~4194304,188743680~4194304,213909504~4194304,226492416~8388608,243269632~8388608,260046848~12582912,289406976~33554432,348127232~8388608,390070272~12582912] 2024-03-29T05:09:57.200 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 6 diff [25165824~4194304,46137344~8388608,62914560~8388608,96468992~12582912,117440512~16777216,146800640~16777216,188743680~4194304,213909504~4194304,226492416~8388608,243269632~8388608,260046848~8388608,276824064~4194304,285212672~12582912,301989888~8388608,314572800~8388608,348127232~8388608,360710144~8388608,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.200 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~4194304,46137344~8388608,62914560~8388608,96468992~12582912,117440512~16777216,146800640~16777216,188743680~4194304,213909504~4194304,226492416~8388608,243269632~8388608,260046848~8388608,276824064~4194304,285212672~12582912,301989888~8388608,314572800~8388608,348127232~8388608,360710144~8388608,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.202 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~4194304,46137344~8388608,62914560~8388608,96468992~16777216,117440512~16777216,146800640~16777216,188743680~4194304,213909504~4194304,226492416~8388608,243269632~8388608,260046848~12582912,276824064~4194304,285212672~41943040,348127232~8388608,360710144~12582912,390070272~12582912,406847488~8388608] 2024-03-29T05:09:57.202 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 7 diff [0~4194304,25165824~12582912,46137344~8388608,58720256~12582912,96468992~12582912,117440512~16777216,146800640~16777216,188743680~4194304,213909504~4194304,226492416~8388608,243269632~12582912,260046848~8388608,276824064~4194304,285212672~12582912,301989888~20971520,348127232~20971520,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.202 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,25165824~12582912,46137344~8388608,58720256~12582912,96468992~12582912,117440512~16777216,146800640~16777216,188743680~4194304,213909504~4194304,226492416~8388608,243269632~12582912,260046848~8388608,276824064~4194304,285212672~12582912,301989888~20971520,348127232~20971520,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:56 smithi181 ceph-mon[106466]: pgmap v1148: 105 pgs: 105 active+clean; 736 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 242 KiB/s rd, 52 MiB/s wr, 381 op/s 2024-03-29T05:09:57.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:56 smithi181 ceph-mon[106466]: osdmap e946: 8 total, 8 up, 8 in 2024-03-29T05:09:57.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:56 smithi181 ceph-mon[106466]: failed to encode map e946 with expected crc 2024-03-29T05:09:57.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:56 smithi181 ceph-mon[106466]: failed to encode map e946 with expected crc 2024-03-29T05:09:57.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:09:57.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:56 smithi181 ceph-mon[101970]: pgmap v1148: 105 pgs: 105 active+clean; 736 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 242 KiB/s rd, 52 MiB/s wr, 381 op/s 2024-03-29T05:09:57.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:56 smithi181 ceph-mon[101970]: osdmap e946: 8 total, 8 up, 8 in 2024-03-29T05:09:57.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:56 smithi181 ceph-mon[101970]: failed to encode map e946 with expected crc 2024-03-29T05:09:57.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:56 smithi181 ceph-mon[101970]: failed to encode map e946 with expected crc 2024-03-29T05:09:57.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:09:57.204 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,25165824~16777216,46137344~8388608,58720256~12582912,96468992~16777216,117440512~16777216,146800640~16777216,188743680~4194304,213909504~4194304,226492416~12582912,243269632~29360128,276824064~4194304,285212672~41943040,348127232~25165824,390070272~12582912,406847488~8388608] 2024-03-29T05:09:57.204 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 8 diff [0~4194304,25165824~16777216,46137344~8388608,58720256~12582912,79691776~8388608,96468992~12582912,117440512~16777216,146800640~16777216,188743680~4194304,213909504~4194304,226492416~8388608,243269632~12582912,260046848~8388608,276824064~4194304,285212672~37748736,348127232~25165824,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.204 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,25165824~16777216,46137344~8388608,58720256~12582912,79691776~8388608,96468992~12582912,117440512~16777216,146800640~16777216,188743680~4194304,213909504~4194304,226492416~8388608,243269632~12582912,260046848~8388608,276824064~4194304,285212672~37748736,348127232~25165824,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.206 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,25165824~29360128,58720256~12582912,79691776~12582912,96468992~16777216,117440512~20971520,146800640~16777216,188743680~4194304,213909504~4194304,226492416~12582912,243269632~29360128,276824064~4194304,285212672~41943040,348127232~25165824,390070272~12582912,406847488~8388608] 2024-03-29T05:09:57.206 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 9 diff [0~4194304,8388608~12582912,25165824~16777216,46137344~25165824,79691776~8388608,96468992~12582912,117440512~20971520,146800640~16777216,180355072~8388608,213909504~4194304,226492416~8388608,243269632~12582912,260046848~8388608,276824064~4194304,285212672~37748736,348127232~25165824,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.207 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~16777216,46137344~25165824,79691776~8388608,96468992~12582912,117440512~20971520,146800640~16777216,180355072~8388608,213909504~4194304,226492416~8388608,243269632~12582912,260046848~8388608,276824064~4194304,285212672~37748736,348127232~25165824,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.210 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~62914560,79691776~12582912,96468992~16777216,117440512~25165824,146800640~16777216,180355072~12582912,213909504~4194304,226492416~12582912,243269632~29360128,276824064~4194304,285212672~41943040,348127232~25165824,390070272~25165824] 2024-03-29T05:09:57.210 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 10 diff [0~4194304,8388608~12582912,25165824~16777216,46137344~25165824,79691776~8388608,96468992~12582912,117440512~20971520,146800640~16777216,180355072~12582912,213909504~4194304,226492416~8388608,243269632~25165824,276824064~4194304,285212672~58720256,348127232~25165824,377487360~4194304,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.210 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~16777216,46137344~25165824,79691776~8388608,96468992~12582912,117440512~20971520,146800640~16777216,180355072~12582912,213909504~4194304,226492416~8388608,243269632~25165824,276824064~4194304,285212672~58720256,348127232~25165824,377487360~4194304,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.213 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~62914560,79691776~12582912,96468992~16777216,117440512~25165824,146800640~16777216,180355072~12582912,213909504~4194304,226492416~12582912,243269632~29360128,276824064~4194304,285212672~58720256,348127232~25165824,377487360~8388608,390070272~25165824] 2024-03-29T05:09:57.213 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 11 diff [0~4194304,8388608~12582912,25165824~16777216,46137344~41943040,96468992~12582912,117440512~25165824,146800640~16777216,180355072~12582912,213909504~4194304,226492416~8388608,243269632~25165824,276824064~4194304,285212672~54525952,348127232~25165824,377487360~4194304,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.213 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~16777216,46137344~41943040,96468992~12582912,117440512~25165824,146800640~16777216,180355072~12582912,213909504~4194304,226492416~8388608,243269632~25165824,276824064~4194304,285212672~54525952,348127232~25165824,377487360~4194304,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.216 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~83886080,96468992~16777216,117440512~25165824,146800640~16777216,180355072~16777216,213909504~8388608,226492416~12582912,243269632~29360128,276824064~4194304,285212672~58720256,348127232~25165824,377487360~8388608,390070272~25165824] 2024-03-29T05:09:57.217 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 12 diff [0~4194304,8388608~12582912,25165824~16777216,46137344~41943040,96468992~12582912,117440512~25165824,146800640~16777216,180355072~12582912,213909504~4194304,226492416~12582912,243269632~25165824,276824064~4194304,285212672~54525952,348127232~25165824,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.217 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~16777216,46137344~41943040,96468992~12582912,117440512~25165824,146800640~16777216,180355072~12582912,213909504~4194304,226492416~12582912,243269632~25165824,276824064~4194304,285212672~54525952,348127232~25165824,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.221 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~83886080,96468992~16777216,117440512~25165824,146800640~16777216,180355072~16777216,213909504~8388608,226492416~46137344,276824064~4194304,285212672~58720256,348127232~67108864] 2024-03-29T05:09:57.221 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 13 diff [0~4194304,8388608~33554432,46137344~41943040,96468992~12582912,117440512~25165824,146800640~16777216,171966464~20971520,213909504~4194304,226492416~12582912,243269632~25165824,276824064~4194304,285212672~54525952,348127232~25165824,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.221 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~33554432,46137344~41943040,96468992~12582912,117440512~25165824,146800640~16777216,171966464~20971520,213909504~4194304,226492416~12582912,243269632~25165824,276824064~4194304,285212672~54525952,348127232~25165824,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.225 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~83886080,96468992~16777216,117440512~25165824,146800640~16777216,171966464~25165824,213909504~8388608,226492416~46137344,276824064~4194304,285212672~58720256,348127232~67108864] 2024-03-29T05:09:57.225 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 14 diff [0~41943040,46137344~41943040,96468992~12582912,117440512~25165824,146800640~16777216,171966464~20971520,205520896~4194304,213909504~4194304,226492416~12582912,243269632~25165824,276824064~4194304,285212672~54525952,343932928~29360128,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.225 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,96468992~12582912,117440512~25165824,146800640~16777216,171966464~20971520,205520896~4194304,213909504~4194304,226492416~12582912,243269632~25165824,276824064~4194304,285212672~54525952,343932928~29360128,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.229 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~92274688,96468992~16777216,117440512~25165824,146800640~16777216,171966464~25165824,205520896~4194304,213909504~8388608,226492416~46137344,276824064~4194304,285212672~130023424] 2024-03-29T05:09:57.229 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 15 diff [0~41943040,46137344~41943040,96468992~46137344,146800640~16777216,171966464~20971520,205520896~4194304,213909504~8388608,226492416~12582912,243269632~29360128,276824064~4194304,285212672~54525952,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.230 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,96468992~46137344,146800640~16777216,171966464~20971520,205520896~4194304,213909504~8388608,226492416~12582912,243269632~29360128,276824064~4194304,285212672~54525952,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.235 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~92274688,96468992~46137344,146800640~16777216,171966464~25165824,205520896~4194304,213909504~67108864,285212672~130023424] 2024-03-29T05:09:57.235 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 16 diff [0~41943040,46137344~41943040,96468992~46137344,146800640~16777216,171966464~20971520,205520896~16777216,226492416~12582912,243269632~29360128,276824064~4194304,285212672~54525952,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.235 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,96468992~46137344,146800640~16777216,171966464~20971520,205520896~16777216,226492416~12582912,243269632~29360128,276824064~4194304,285212672~54525952,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.240 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~92274688,96468992~46137344,146800640~16777216,171966464~25165824,205520896~75497472,285212672~130023424] 2024-03-29T05:09:57.240 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 17 diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,205520896~16777216,226492416~12582912,243269632~29360128,276824064~4194304,285212672~54525952,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.241 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,205520896~16777216,226492416~12582912,243269632~29360128,276824064~4194304,285212672~54525952,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.246 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~25165824,205520896~75497472,285212672~130023424] 2024-03-29T05:09:57.246 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 18 diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.246 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.252 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~25165824,205520896~75497472,285212672~130023424] 2024-03-29T05:09:57.252 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to 19 diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:09:57.252 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:09:57.259 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:09:57.259 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 5 diff [46137344~4194304,100663296~8388608,117440512~12582912,188743680~4194304,213909504~4194304,243269632~8388608,314572800~8388608,390070272~4194304,398458880~4194304] 2024-03-29T05:09:57.259 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [46137344~4194304,100663296~8388608,117440512~12582912,188743680~4194304,213909504~4194304,243269632~8388608,314572800~8388608,390070272~4194304,398458880~4194304] 2024-03-29T05:09:57.260 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [46137344~4194304,100663296~12582912,117440512~16777216,188743680~4194304,213909504~4194304,243269632~8388608,314572800~8388608,390070272~12582912] 2024-03-29T05:09:57.261 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 6 diff [25165824~4194304,46137344~4194304,100663296~8388608,117440512~16777216,146800640~16777216,188743680~4194304,213909504~4194304,243269632~8388608,276824064~4194304,285212672~8388608,314572800~8388608,360710144~8388608,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.261 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~4194304,46137344~4194304,100663296~8388608,117440512~16777216,146800640~16777216,188743680~4194304,213909504~4194304,243269632~8388608,276824064~4194304,285212672~8388608,314572800~8388608,360710144~8388608,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.262 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~4194304,46137344~4194304,100663296~12582912,117440512~16777216,146800640~16777216,188743680~4194304,213909504~4194304,243269632~8388608,276824064~4194304,285212672~12582912,314572800~12582912,360710144~12582912,390070272~12582912,406847488~8388608] 2024-03-29T05:09:57.262 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 7 diff [0~4194304,25165824~12582912,46137344~4194304,58720256~4194304,100663296~8388608,117440512~16777216,146800640~16777216,188743680~4194304,213909504~4194304,226492416~8388608,243269632~12582912,276824064~4194304,285212672~8388608,301989888~8388608,314572800~8388608,348127232~8388608,360710144~8388608,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.262 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,25165824~12582912,46137344~4194304,58720256~4194304,100663296~8388608,117440512~16777216,146800640~16777216,188743680~4194304,213909504~4194304,226492416~8388608,243269632~12582912,276824064~4194304,285212672~8388608,301989888~8388608,314572800~8388608,348127232~8388608,360710144~8388608,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.264 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,25165824~16777216,46137344~4194304,58720256~4194304,100663296~12582912,117440512~16777216,146800640~16777216,188743680~4194304,213909504~4194304,226492416~12582912,243269632~16777216,276824064~4194304,285212672~12582912,301989888~25165824,348127232~25165824,390070272~12582912,406847488~8388608] 2024-03-29T05:09:57.264 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 8 diff [0~4194304,25165824~16777216,46137344~4194304,58720256~4194304,79691776~8388608,100663296~8388608,117440512~16777216,146800640~16777216,188743680~4194304,213909504~4194304,226492416~8388608,243269632~12582912,264241152~4194304,276824064~4194304,285212672~8388608,297795584~12582912,314572800~8388608,348127232~8388608,360710144~12582912,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.264 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,25165824~16777216,46137344~4194304,58720256~4194304,79691776~8388608,100663296~8388608,117440512~16777216,146800640~16777216,188743680~4194304,213909504~4194304,226492416~8388608,243269632~12582912,264241152~4194304,276824064~4194304,285212672~8388608,297795584~12582912,314572800~8388608,348127232~8388608,360710144~12582912,390070272~4194304,398458880~4194304,406847488~4194304] 2024-03-29T05:09:57.266 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,25165824~25165824,58720256~4194304,79691776~12582912,100663296~12582912,117440512~20971520,146800640~16777216,188743680~4194304,213909504~4194304,226492416~12582912,243269632~16777216,264241152~4194304,276824064~4194304,285212672~41943040,348127232~25165824,390070272~12582912,406847488~8388608] 2024-03-29T05:09:57.266 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 9 diff [0~4194304,8388608~12582912,25165824~16777216,46137344~4194304,54525952~8388608,79691776~8388608,100663296~8388608,117440512~20971520,146800640~16777216,180355072~8388608,213909504~4194304,226492416~8388608,243269632~12582912,264241152~4194304,276824064~4194304,285212672~8388608,297795584~12582912,314572800~8388608,348127232~8388608,360710144~12582912,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.266 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~16777216,46137344~4194304,54525952~8388608,79691776~8388608,100663296~8388608,117440512~20971520,146800640~16777216,180355072~8388608,213909504~4194304,226492416~8388608,243269632~12582912,264241152~4194304,276824064~4194304,285212672~8388608,297795584~12582912,314572800~8388608,348127232~8388608,360710144~12582912,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.268 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~41943040,54525952~8388608,79691776~12582912,100663296~12582912,117440512~25165824,146800640~16777216,180355072~12582912,213909504~4194304,226492416~12582912,243269632~16777216,264241152~4194304,276824064~4194304,285212672~41943040,348127232~25165824,390070272~25165824] 2024-03-29T05:09:57.269 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 10 diff [0~4194304,8388608~12582912,25165824~16777216,46137344~4194304,54525952~8388608,79691776~8388608,100663296~8388608,117440512~20971520,146800640~16777216,180355072~12582912,213909504~4194304,226492416~8388608,243269632~16777216,264241152~4194304,276824064~4194304,285212672~8388608,297795584~46137344,348127232~8388608,360710144~12582912,377487360~4194304,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.269 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~16777216,46137344~4194304,54525952~8388608,79691776~8388608,100663296~8388608,117440512~20971520,146800640~16777216,180355072~12582912,213909504~4194304,226492416~8388608,243269632~16777216,264241152~4194304,276824064~4194304,285212672~8388608,297795584~46137344,348127232~8388608,360710144~12582912,377487360~4194304,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.272 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~41943040,54525952~8388608,79691776~12582912,100663296~12582912,117440512~25165824,146800640~16777216,180355072~12582912,213909504~4194304,226492416~12582912,243269632~16777216,264241152~4194304,276824064~4194304,285212672~58720256,348127232~25165824,377487360~8388608,390070272~25165824] 2024-03-29T05:09:57.272 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 11 diff [0~4194304,8388608~12582912,25165824~16777216,46137344~16777216,71303168~16777216,100663296~8388608,117440512~25165824,146800640~16777216,180355072~12582912,213909504~4194304,226492416~8388608,243269632~16777216,264241152~4194304,276824064~4194304,285212672~8388608,297795584~41943040,348127232~8388608,360710144~12582912,377487360~4194304,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.272 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~16777216,46137344~16777216,71303168~16777216,100663296~8388608,117440512~25165824,146800640~16777216,180355072~12582912,213909504~4194304,226492416~8388608,243269632~16777216,264241152~4194304,276824064~4194304,285212672~8388608,297795584~41943040,348127232~8388608,360710144~12582912,377487360~4194304,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.276 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~54525952,71303168~20971520,100663296~12582912,117440512~25165824,146800640~16777216,180355072~16777216,213909504~8388608,226492416~12582912,243269632~16777216,264241152~4194304,276824064~4194304,285212672~58720256,348127232~25165824,377487360~8388608,390070272~25165824] 2024-03-29T05:09:57.276 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 12 diff [0~4194304,8388608~12582912,25165824~16777216,46137344~16777216,71303168~16777216,100663296~8388608,117440512~25165824,146800640~16777216,180355072~12582912,213909504~4194304,226492416~12582912,243269632~16777216,264241152~4194304,276824064~4194304,285212672~8388608,297795584~41943040,348127232~25165824,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.276 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~16777216,46137344~16777216,71303168~16777216,100663296~8388608,117440512~25165824,146800640~16777216,180355072~12582912,213909504~4194304,226492416~12582912,243269632~16777216,264241152~4194304,276824064~4194304,285212672~8388608,297795584~41943040,348127232~25165824,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.279 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~54525952,71303168~20971520,100663296~12582912,117440512~25165824,146800640~16777216,180355072~16777216,213909504~8388608,226492416~33554432,264241152~4194304,276824064~4194304,285212672~58720256,348127232~67108864] 2024-03-29T05:09:57.280 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 13 diff [0~4194304,8388608~33554432,46137344~16777216,71303168~16777216,100663296~8388608,117440512~25165824,146800640~16777216,171966464~20971520,213909504~4194304,226492416~12582912,243269632~16777216,264241152~8388608,276824064~4194304,285212672~8388608,297795584~41943040,348127232~25165824,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.280 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~33554432,46137344~16777216,71303168~16777216,100663296~8388608,117440512~25165824,146800640~16777216,171966464~20971520,213909504~4194304,226492416~12582912,243269632~16777216,264241152~8388608,276824064~4194304,285212672~8388608,297795584~41943040,348127232~25165824,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.284 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~54525952,71303168~20971520,100663296~12582912,117440512~25165824,146800640~16777216,171966464~25165824,213909504~8388608,226492416~33554432,264241152~8388608,276824064~4194304,285212672~58720256,348127232~67108864] 2024-03-29T05:09:57.284 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 14 diff [0~41943040,46137344~16777216,71303168~16777216,100663296~8388608,117440512~25165824,146800640~16777216,171966464~20971520,205520896~4194304,213909504~4194304,226492416~12582912,243269632~16777216,264241152~8388608,276824064~4194304,285212672~8388608,297795584~41943040,343932928~29360128,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.284 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,71303168~16777216,100663296~8388608,117440512~25165824,146800640~16777216,171966464~20971520,205520896~4194304,213909504~4194304,226492416~12582912,243269632~16777216,264241152~8388608,276824064~4194304,285212672~8388608,297795584~41943040,343932928~29360128,377487360~8388608,390070272~4194304,398458880~12582912] 2024-03-29T05:09:57.287 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,71303168~20971520,100663296~12582912,117440512~25165824,146800640~16777216,171966464~25165824,205520896~4194304,213909504~8388608,226492416~33554432,264241152~8388608,276824064~4194304,285212672~130023424] 2024-03-29T05:09:57.287 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 15 diff [0~41943040,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~4194304,213909504~8388608,226492416~12582912,243269632~29360128,276824064~4194304,285212672~8388608,297795584~41943040,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.288 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~4194304,213909504~8388608,226492416~12582912,243269632~29360128,276824064~4194304,285212672~8388608,297795584~41943040,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.293 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~25165824,100663296~41943040,146800640~16777216,171966464~25165824,205520896~4194304,213909504~67108864,285212672~130023424] 2024-03-29T05:09:57.293 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 16 diff [0~41943040,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~16777216,226492416~12582912,243269632~29360128,276824064~4194304,285212672~8388608,297795584~41943040,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.293 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~16777216,226492416~12582912,243269632~29360128,276824064~4194304,285212672~8388608,297795584~41943040,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.298 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~25165824,100663296~41943040,146800640~16777216,171966464~25165824,205520896~75497472,285212672~130023424] 2024-03-29T05:09:57.298 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 17 diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,205520896~16777216,226492416~12582912,243269632~29360128,276824064~4194304,285212672~8388608,297795584~41943040,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.298 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,205520896~16777216,226492416~12582912,243269632~29360128,276824064~4194304,285212672~8388608,297795584~41943040,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.303 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~25165824,205520896~75497472,285212672~130023424] 2024-03-29T05:09:57.303 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 18 diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.304 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~29360128,377487360~16777216,398458880~12582912] 2024-03-29T05:09:57.311 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~25165824,205520896~75497472,285212672~130023424] 2024-03-29T05:09:57.311 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to 19 diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:09:57.311 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:09:57.318 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~109051904,285212672~130023424] 2024-03-29T05:09:57.318 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 6 diff [25165824~4194304,125829120~8388608,146800640~16777216,276824064~4194304,285212672~8388608,318767104~4194304,360710144~8388608,406847488~4194304] 2024-03-29T05:09:57.318 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~4194304,125829120~8388608,146800640~16777216,276824064~4194304,285212672~8388608,318767104~4194304,360710144~8388608,406847488~4194304] 2024-03-29T05:09:57.319 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~4194304,125829120~8388608,146800640~16777216,276824064~4194304,285212672~12582912,318767104~8388608,360710144~12582912,406847488~8388608] 2024-03-29T05:09:57.319 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 7 diff [0~4194304,25165824~12582912,46137344~4194304,58720256~4194304,125829120~8388608,146800640~16777216,226492416~8388608,251658240~4194304,276824064~4194304,285212672~8388608,301989888~8388608,318767104~4194304,348127232~8388608,360710144~8388608,406847488~4194304] 2024-03-29T05:09:57.319 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,25165824~12582912,46137344~4194304,58720256~4194304,125829120~8388608,146800640~16777216,226492416~8388608,251658240~4194304,276824064~4194304,285212672~8388608,301989888~8388608,318767104~4194304,348127232~8388608,360710144~8388608,406847488~4194304] 2024-03-29T05:09:57.321 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,25165824~16777216,46137344~4194304,58720256~4194304,125829120~8388608,146800640~16777216,226492416~12582912,251658240~8388608,276824064~4194304,285212672~12582912,301989888~12582912,318767104~8388608,348127232~25165824,406847488~8388608] 2024-03-29T05:09:57.321 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 8 diff [0~4194304,25165824~16777216,46137344~4194304,58720256~4194304,79691776~8388608,125829120~8388608,146800640~16777216,226492416~8388608,251658240~4194304,264241152~4194304,276824064~4194304,285212672~8388608,297795584~12582912,318767104~4194304,348127232~8388608,360710144~12582912,406847488~4194304] 2024-03-29T05:09:57.321 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,25165824~16777216,46137344~4194304,58720256~4194304,79691776~8388608,125829120~8388608,146800640~16777216,226492416~8388608,251658240~4194304,264241152~4194304,276824064~4194304,285212672~8388608,297795584~12582912,318767104~4194304,348127232~8388608,360710144~12582912,406847488~4194304] 2024-03-29T05:09:57.323 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,25165824~25165824,58720256~4194304,79691776~12582912,125829120~12582912,146800640~16777216,226492416~12582912,251658240~8388608,264241152~4194304,276824064~4194304,285212672~29360128,318767104~8388608,348127232~25165824,406847488~8388608] 2024-03-29T05:09:57.323 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 9 diff [0~4194304,8388608~12582912,25165824~16777216,46137344~4194304,54525952~8388608,79691776~8388608,100663296~8388608,125829120~12582912,146800640~16777216,180355072~8388608,226492416~8388608,251658240~4194304,264241152~4194304,276824064~4194304,285212672~8388608,297795584~12582912,318767104~4194304,348127232~8388608,360710144~12582912,398458880~12582912] 2024-03-29T05:09:57.323 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~16777216,46137344~4194304,54525952~8388608,79691776~8388608,100663296~8388608,125829120~12582912,146800640~16777216,180355072~8388608,226492416~8388608,251658240~4194304,264241152~4194304,276824064~4194304,285212672~8388608,297795584~12582912,318767104~4194304,348127232~8388608,360710144~12582912,398458880~12582912] 2024-03-29T05:09:57.325 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~41943040,54525952~8388608,79691776~12582912,100663296~12582912,125829120~16777216,146800640~16777216,180355072~12582912,226492416~12582912,251658240~8388608,264241152~4194304,276824064~4194304,285212672~29360128,318767104~8388608,348127232~25165824,398458880~16777216] 2024-03-29T05:09:57.325 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 10 diff [0~4194304,8388608~12582912,25165824~16777216,46137344~4194304,54525952~8388608,79691776~8388608,100663296~8388608,125829120~12582912,146800640~16777216,180355072~12582912,226492416~8388608,247463936~12582912,264241152~4194304,276824064~4194304,285212672~8388608,297795584~16777216,318767104~25165824,348127232~8388608,360710144~12582912,377487360~4194304,398458880~12582912] 2024-03-29T05:09:57.326 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~16777216,46137344~4194304,54525952~8388608,79691776~8388608,100663296~8388608,125829120~12582912,146800640~16777216,180355072~12582912,226492416~8388608,247463936~12582912,264241152~4194304,276824064~4194304,285212672~8388608,297795584~16777216,318767104~25165824,348127232~8388608,360710144~12582912,377487360~4194304,398458880~12582912] 2024-03-29T05:09:57.329 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~41943040,54525952~8388608,79691776~12582912,100663296~12582912,125829120~16777216,146800640~16777216,180355072~12582912,226492416~12582912,247463936~12582912,264241152~4194304,276824064~4194304,285212672~29360128,318767104~25165824,348127232~25165824,377487360~8388608,398458880~16777216] 2024-03-29T05:09:57.329 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 11 diff [0~4194304,8388608~12582912,25165824~16777216,46137344~16777216,71303168~16777216,100663296~8388608,121634816~20971520,146800640~16777216,180355072~12582912,213909504~4194304,226492416~8388608,247463936~12582912,264241152~4194304,276824064~4194304,285212672~8388608,297795584~16777216,318767104~20971520,348127232~8388608,360710144~12582912,377487360~4194304,398458880~12582912] 2024-03-29T05:09:57.329 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~16777216,46137344~16777216,71303168~16777216,100663296~8388608,121634816~20971520,146800640~16777216,180355072~12582912,213909504~4194304,226492416~8388608,247463936~12582912,264241152~4194304,276824064~4194304,285212672~8388608,297795584~16777216,318767104~20971520,348127232~8388608,360710144~12582912,377487360~4194304,398458880~12582912] 2024-03-29T05:09:57.331 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~54525952,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,180355072~16777216,213909504~8388608,226492416~12582912,247463936~12582912,264241152~4194304,276824064~4194304,285212672~29360128,318767104~25165824,348127232~25165824,377487360~8388608,398458880~16777216] 2024-03-29T05:09:57.331 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 12 diff [0~4194304,8388608~12582912,25165824~16777216,46137344~16777216,71303168~16777216,100663296~8388608,121634816~20971520,146800640~16777216,180355072~12582912,213909504~4194304,226492416~12582912,247463936~12582912,264241152~4194304,276824064~4194304,285212672~8388608,297795584~16777216,318767104~20971520,348127232~25165824,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.331 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,25165824~16777216,46137344~16777216,71303168~16777216,100663296~8388608,121634816~20971520,146800640~16777216,180355072~12582912,213909504~4194304,226492416~12582912,247463936~12582912,264241152~4194304,276824064~4194304,285212672~8388608,297795584~16777216,318767104~20971520,348127232~25165824,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.335 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~54525952,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,180355072~16777216,213909504~8388608,226492416~16777216,247463936~12582912,264241152~4194304,276824064~4194304,285212672~29360128,318767104~25165824,348127232~41943040,398458880~16777216] 2024-03-29T05:09:57.335 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 13 diff [0~4194304,8388608~33554432,46137344~16777216,71303168~16777216,100663296~8388608,121634816~20971520,146800640~16777216,171966464~20971520,213909504~4194304,226492416~12582912,247463936~12582912,264241152~8388608,276824064~4194304,285212672~8388608,297795584~16777216,318767104~20971520,348127232~25165824,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.335 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~33554432,46137344~16777216,71303168~16777216,100663296~8388608,121634816~20971520,146800640~16777216,171966464~20971520,213909504~4194304,226492416~12582912,247463936~12582912,264241152~8388608,276824064~4194304,285212672~8388608,297795584~16777216,318767104~20971520,348127232~25165824,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.341 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~54525952,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,171966464~25165824,213909504~8388608,226492416~16777216,247463936~12582912,264241152~8388608,276824064~4194304,285212672~29360128,318767104~25165824,348127232~41943040,398458880~16777216] 2024-03-29T05:09:57.341 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 14 diff [0~41943040,46137344~16777216,71303168~16777216,100663296~8388608,121634816~20971520,146800640~16777216,171966464~20971520,205520896~4194304,213909504~4194304,226492416~12582912,247463936~12582912,264241152~8388608,276824064~4194304,285212672~8388608,297795584~16777216,318767104~20971520,343932928~29360128,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.341 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,71303168~16777216,100663296~8388608,121634816~20971520,146800640~16777216,171966464~20971520,205520896~4194304,213909504~4194304,226492416~12582912,247463936~12582912,264241152~8388608,276824064~4194304,285212672~8388608,297795584~16777216,318767104~20971520,343932928~29360128,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.343 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,171966464~25165824,205520896~4194304,213909504~8388608,226492416~16777216,247463936~12582912,264241152~8388608,276824064~4194304,285212672~29360128,318767104~71303168,398458880~16777216] 2024-03-29T05:09:57.343 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 15 diff [0~41943040,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~4194304,213909504~8388608,226492416~12582912,247463936~25165824,276824064~4194304,285212672~8388608,297795584~41943040,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.343 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~4194304,213909504~8388608,226492416~12582912,247463936~25165824,276824064~4194304,285212672~8388608,297795584~41943040,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.347 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~25165824,100663296~41943040,146800640~16777216,171966464~25165824,205520896~4194304,213909504~29360128,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:09:57.347 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 16 diff [0~41943040,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~16777216,226492416~12582912,247463936~25165824,276824064~4194304,285212672~8388608,297795584~41943040,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.347 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~16777216,226492416~12582912,247463936~25165824,276824064~4194304,285212672~8388608,297795584~41943040,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.352 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~25165824,100663296~41943040,146800640~16777216,171966464~25165824,205520896~37748736,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:09:57.352 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 17 diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,205520896~16777216,226492416~12582912,247463936~25165824,276824064~4194304,285212672~8388608,297795584~41943040,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.352 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,205520896~16777216,226492416~12582912,247463936~25165824,276824064~4194304,285212672~8388608,297795584~41943040,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.357 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~25165824,205520896~37748736,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:09:57.357 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 18 diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.357 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.362 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~25165824,205520896~37748736,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:09:57.362 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to 19 diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:09:57.362 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:09:57.368 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:09:57.368 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 7 diff [0~4194304,29360128~8388608,46137344~4194304,58720256~4194304,150994944~8388608,226492416~8388608,251658240~4194304,301989888~8388608,348127232~8388608] 2024-03-29T05:09:57.369 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,29360128~8388608,46137344~4194304,58720256~4194304,150994944~8388608,226492416~8388608,251658240~4194304,301989888~8388608,348127232~8388608] 2024-03-29T05:09:57.369 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,29360128~12582912,46137344~4194304,58720256~4194304,150994944~12582912,226492416~12582912,251658240~8388608,301989888~12582912,348127232~12582912] 2024-03-29T05:09:57.369 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 8 diff [0~4194304,29360128~12582912,46137344~4194304,58720256~4194304,79691776~8388608,130023424~4194304,146800640~12582912,226492416~8388608,251658240~4194304,264241152~4194304,297795584~12582912,348127232~8388608,369098752~4194304] 2024-03-29T05:09:57.369 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,29360128~12582912,46137344~4194304,58720256~4194304,79691776~8388608,130023424~4194304,146800640~12582912,226492416~8388608,251658240~4194304,264241152~4194304,297795584~12582912,348127232~8388608,369098752~4194304] 2024-03-29T05:09:57.371 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,29360128~20971520,58720256~4194304,79691776~12582912,130023424~8388608,146800640~16777216,226492416~12582912,251658240~8388608,264241152~4194304,297795584~16777216,348127232~12582912,369098752~4194304] 2024-03-29T05:09:57.371 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 9 diff [0~4194304,8388608~12582912,29360128~12582912,46137344~4194304,54525952~8388608,79691776~8388608,100663296~8388608,130023424~8388608,146800640~12582912,180355072~8388608,226492416~8388608,251658240~4194304,264241152~4194304,297795584~12582912,348127232~8388608,369098752~4194304,398458880~12582912] 2024-03-29T05:09:57.371 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,29360128~12582912,46137344~4194304,54525952~8388608,79691776~8388608,100663296~8388608,130023424~8388608,146800640~12582912,180355072~8388608,226492416~8388608,251658240~4194304,264241152~4194304,297795584~12582912,348127232~8388608,369098752~4194304,398458880~12582912] 2024-03-29T05:09:57.372 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~16777216,29360128~20971520,54525952~8388608,79691776~12582912,100663296~12582912,130023424~12582912,146800640~16777216,180355072~12582912,226492416~12582912,251658240~8388608,264241152~4194304,297795584~16777216,348127232~12582912,369098752~4194304,398458880~16777216] 2024-03-29T05:09:57.373 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 10 diff [0~4194304,8388608~12582912,29360128~12582912,46137344~4194304,54525952~8388608,79691776~8388608,100663296~8388608,130023424~8388608,146800640~12582912,180355072~12582912,226492416~8388608,247463936~12582912,264241152~4194304,297795584~16777216,322961408~20971520,348127232~8388608,369098752~4194304,377487360~4194304,398458880~12582912] 2024-03-29T05:09:57.373 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,29360128~12582912,46137344~4194304,54525952~8388608,79691776~8388608,100663296~8388608,130023424~8388608,146800640~12582912,180355072~12582912,226492416~8388608,247463936~12582912,264241152~4194304,297795584~16777216,322961408~20971520,348127232~8388608,369098752~4194304,377487360~4194304,398458880~12582912] 2024-03-29T05:09:57.375 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~16777216,29360128~20971520,54525952~8388608,79691776~12582912,100663296~12582912,130023424~12582912,146800640~16777216,180355072~12582912,226492416~12582912,247463936~12582912,264241152~4194304,297795584~16777216,322961408~20971520,348127232~12582912,369098752~4194304,377487360~8388608,398458880~16777216] 2024-03-29T05:09:57.375 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 11 diff [0~4194304,8388608~12582912,29360128~12582912,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,146800640~16777216,180355072~12582912,213909504~4194304,226492416~8388608,247463936~12582912,264241152~4194304,297795584~16777216,322961408~16777216,348127232~8388608,364904448~8388608,377487360~4194304,398458880~12582912] 2024-03-29T05:09:57.375 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,29360128~12582912,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,146800640~16777216,180355072~12582912,213909504~4194304,226492416~8388608,247463936~12582912,264241152~4194304,297795584~16777216,322961408~16777216,348127232~8388608,364904448~8388608,377487360~4194304,398458880~12582912] 2024-03-29T05:09:57.377 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~16777216,29360128~33554432,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,180355072~16777216,213909504~8388608,226492416~12582912,247463936~12582912,264241152~4194304,297795584~16777216,322961408~20971520,348127232~12582912,364904448~8388608,377487360~8388608,398458880~16777216] 2024-03-29T05:09:57.378 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 12 diff [0~4194304,8388608~12582912,29360128~12582912,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,146800640~16777216,180355072~12582912,213909504~4194304,226492416~12582912,247463936~12582912,264241152~4194304,297795584~16777216,322961408~16777216,348127232~12582912,364904448~8388608,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.378 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~12582912,29360128~12582912,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,146800640~16777216,180355072~12582912,213909504~4194304,226492416~12582912,247463936~12582912,264241152~4194304,297795584~16777216,322961408~16777216,348127232~12582912,364904448~8388608,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.381 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~16777216,29360128~33554432,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,180355072~16777216,213909504~8388608,226492416~16777216,247463936~12582912,264241152~4194304,297795584~16777216,322961408~20971520,348127232~41943040,398458880~16777216] 2024-03-29T05:09:57.381 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 13 diff [0~4194304,8388608~16777216,29360128~12582912,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,146800640~16777216,171966464~20971520,213909504~4194304,226492416~12582912,247463936~12582912,264241152~8388608,297795584~16777216,322961408~16777216,348127232~12582912,364904448~8388608,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.381 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~4194304,8388608~16777216,29360128~12582912,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,146800640~16777216,171966464~20971520,213909504~4194304,226492416~12582912,247463936~12582912,264241152~8388608,297795584~16777216,322961408~16777216,348127232~12582912,364904448~8388608,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:56 smithi187 ceph-mon[93536]: pgmap v1148: 105 pgs: 105 active+clean; 736 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 242 KiB/s rd, 52 MiB/s wr, 381 op/s 2024-03-29T05:09:57.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:56 smithi187 ceph-mon[93536]: osdmap e946: 8 total, 8 up, 8 in 2024-03-29T05:09:57.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:56 smithi187 ceph-mon[93536]: failed to encode map e946 with expected crc 2024-03-29T05:09:57.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:56 smithi187 ceph-mon[93536]: failed to encode map e946 with expected crc 2024-03-29T05:09:57.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:09:57.384 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~4194304,8388608~16777216,29360128~33554432,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,171966464~25165824,213909504~8388608,226492416~16777216,247463936~12582912,264241152~8388608,297795584~16777216,322961408~20971520,348127232~41943040,398458880~16777216] 2024-03-29T05:09:57.384 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 14 diff [0~41943040,46137344~16777216,71303168~16777216,100663296~8388608,121634816~20971520,146800640~16777216,171966464~20971520,205520896~4194304,213909504~4194304,226492416~12582912,247463936~12582912,264241152~8388608,297795584~16777216,318767104~20971520,343932928~29360128,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.385 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,71303168~16777216,100663296~8388608,121634816~20971520,146800640~16777216,171966464~20971520,205520896~4194304,213909504~4194304,226492416~12582912,247463936~12582912,264241152~8388608,297795584~16777216,318767104~20971520,343932928~29360128,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.388 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,171966464~25165824,205520896~4194304,213909504~8388608,226492416~16777216,247463936~12582912,264241152~8388608,297795584~16777216,318767104~71303168,398458880~16777216] 2024-03-29T05:09:57.388 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 15 diff [0~41943040,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~4194304,213909504~8388608,226492416~12582912,247463936~25165824,297795584~41943040,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.388 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~4194304,213909504~8388608,226492416~12582912,247463936~25165824,297795584~41943040,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.391 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~25165824,100663296~41943040,146800640~16777216,171966464~25165824,205520896~4194304,213909504~29360128,247463936~29360128,297795584~92274688,398458880~16777216] 2024-03-29T05:09:57.391 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 16 diff [0~41943040,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~16777216,226492416~12582912,247463936~25165824,297795584~41943040,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.391 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~16777216,226492416~12582912,247463936~25165824,297795584~41943040,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.395 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~25165824,100663296~41943040,146800640~16777216,171966464~25165824,205520896~37748736,247463936~29360128,297795584~92274688,398458880~16777216] 2024-03-29T05:09:57.396 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 17 diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,205520896~16777216,226492416~12582912,247463936~25165824,293601280~46137344,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.396 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,205520896~16777216,226492416~12582912,247463936~25165824,293601280~46137344,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.401 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~25165824,205520896~37748736,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:09:57.401 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 18 diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,205520896~33554432,247463936~25165824,293601280~46137344,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.401 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,205520896~33554432,247463936~25165824,293601280~46137344,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.407 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~25165824,205520896~37748736,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:09:57.407 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to 19 diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:09:57.407 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:09:57.412 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:09:57.412 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 8 diff [37748736~4194304,79691776~8388608,130023424~4194304,146800640~8388608,264241152~4194304,297795584~4194304,348127232~8388608,369098752~4194304] 2024-03-29T05:09:57.412 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [37748736~4194304,79691776~8388608,130023424~4194304,146800640~8388608,264241152~4194304,297795584~4194304,348127232~8388608,369098752~4194304] 2024-03-29T05:09:57.413 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [37748736~8388608,79691776~12582912,130023424~8388608,146800640~8388608,264241152~4194304,297795584~8388608,348127232~12582912,369098752~4194304] 2024-03-29T05:09:57.413 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 9 diff [8388608~12582912,37748736~4194304,54525952~8388608,79691776~8388608,100663296~8388608,130023424~8388608,146800640~8388608,180355072~8388608,264241152~4194304,297795584~4194304,348127232~8388608,369098752~4194304,398458880~12582912] 2024-03-29T05:09:57.413 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~12582912,37748736~4194304,54525952~8388608,79691776~8388608,100663296~8388608,130023424~8388608,146800640~8388608,180355072~8388608,264241152~4194304,297795584~4194304,348127232~8388608,369098752~4194304,398458880~12582912] 2024-03-29T05:09:57.415 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~16777216,37748736~8388608,54525952~8388608,79691776~12582912,100663296~12582912,130023424~12582912,146800640~8388608,180355072~12582912,264241152~4194304,297795584~8388608,348127232~12582912,369098752~4194304,398458880~16777216] 2024-03-29T05:09:57.415 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 10 diff [8388608~12582912,37748736~4194304,54525952~8388608,79691776~8388608,100663296~8388608,130023424~8388608,146800640~8388608,180355072~12582912,247463936~12582912,264241152~4194304,297795584~12582912,322961408~20971520,348127232~8388608,369098752~4194304,377487360~4194304,398458880~12582912] 2024-03-29T05:09:57.415 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~12582912,37748736~4194304,54525952~8388608,79691776~8388608,100663296~8388608,130023424~8388608,146800640~8388608,180355072~12582912,247463936~12582912,264241152~4194304,297795584~12582912,322961408~20971520,348127232~8388608,369098752~4194304,377487360~4194304,398458880~12582912] 2024-03-29T05:09:57.417 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~16777216,37748736~8388608,54525952~8388608,79691776~12582912,100663296~12582912,130023424~12582912,146800640~8388608,180355072~12582912,247463936~12582912,264241152~4194304,297795584~16777216,322961408~20971520,348127232~12582912,369098752~4194304,377487360~8388608,398458880~16777216] 2024-03-29T05:09:57.417 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 11 diff [8388608~12582912,37748736~4194304,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,146800640~16777216,180355072~12582912,213909504~4194304,247463936~12582912,264241152~4194304,297795584~12582912,322961408~16777216,348127232~8388608,364904448~8388608,377487360~4194304,398458880~12582912] 2024-03-29T05:09:57.417 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~12582912,37748736~4194304,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,146800640~16777216,180355072~12582912,213909504~4194304,247463936~12582912,264241152~4194304,297795584~12582912,322961408~16777216,348127232~8388608,364904448~8388608,377487360~4194304,398458880~12582912] 2024-03-29T05:09:57.420 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~16777216,37748736~25165824,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,180355072~16777216,213909504~8388608,247463936~12582912,264241152~4194304,297795584~16777216,322961408~20971520,348127232~12582912,364904448~8388608,377487360~8388608,398458880~16777216] 2024-03-29T05:09:57.420 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 12 diff [8388608~12582912,37748736~4194304,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,146800640~16777216,180355072~12582912,213909504~4194304,230686720~8388608,247463936~12582912,264241152~4194304,297795584~12582912,322961408~16777216,348127232~12582912,364904448~8388608,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.420 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~12582912,37748736~4194304,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,146800640~16777216,180355072~12582912,213909504~4194304,230686720~8388608,247463936~12582912,264241152~4194304,297795584~12582912,322961408~16777216,348127232~12582912,364904448~8388608,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.423 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~16777216,37748736~25165824,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,180355072~16777216,213909504~8388608,230686720~12582912,247463936~12582912,264241152~4194304,297795584~16777216,322961408~20971520,348127232~41943040,398458880~16777216] 2024-03-29T05:09:57.423 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 13 diff [8388608~16777216,29360128~4194304,37748736~4194304,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,146800640~16777216,171966464~20971520,213909504~4194304,230686720~8388608,247463936~12582912,264241152~8388608,297795584~12582912,322961408~16777216,348127232~12582912,364904448~8388608,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.423 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~16777216,29360128~4194304,37748736~4194304,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,146800640~16777216,171966464~20971520,213909504~4194304,230686720~8388608,247463936~12582912,264241152~8388608,297795584~12582912,322961408~16777216,348127232~12582912,364904448~8388608,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.426 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~16777216,29360128~4194304,37748736~25165824,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,171966464~25165824,213909504~8388608,230686720~12582912,247463936~12582912,264241152~8388608,297795584~16777216,322961408~20971520,348127232~41943040,398458880~16777216] 2024-03-29T05:09:57.426 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 14 diff [4194304~29360128,37748736~4194304,46137344~16777216,71303168~16777216,100663296~8388608,121634816~20971520,146800640~16777216,171966464~20971520,205520896~4194304,213909504~4194304,230686720~8388608,247463936~12582912,264241152~8388608,297795584~12582912,318767104~20971520,343932928~29360128,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.426 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~29360128,37748736~4194304,46137344~16777216,71303168~16777216,100663296~8388608,121634816~20971520,146800640~16777216,171966464~20971520,205520896~4194304,213909504~4194304,230686720~8388608,247463936~12582912,264241152~8388608,297795584~12582912,318767104~20971520,343932928~29360128,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.429 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,171966464~25165824,205520896~4194304,213909504~8388608,230686720~12582912,247463936~12582912,264241152~8388608,297795584~16777216,318767104~71303168,398458880~16777216] 2024-03-29T05:09:57.429 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 15 diff [4194304~29360128,37748736~4194304,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~4194304,213909504~8388608,230686720~8388608,247463936~25165824,297795584~12582912,314572800~25165824,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.429 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~29360128,37748736~4194304,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~4194304,213909504~8388608,230686720~8388608,247463936~25165824,297795584~12582912,314572800~25165824,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.433 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~25165824,100663296~41943040,146800640~16777216,171966464~25165824,205520896~4194304,213909504~12582912,230686720~12582912,247463936~29360128,297795584~92274688,398458880~16777216] 2024-03-29T05:09:57.433 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 16 diff [4194304~37748736,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~16777216,230686720~8388608,247463936~25165824,297795584~12582912,314572800~25165824,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.433 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~16777216,230686720~8388608,247463936~25165824,297795584~12582912,314572800~25165824,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.438 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~25165824,100663296~41943040,146800640~16777216,171966464~25165824,205520896~20971520,230686720~12582912,247463936~29360128,297795584~92274688,398458880~16777216] 2024-03-29T05:09:57.438 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 17 diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,205520896~16777216,230686720~8388608,247463936~25165824,293601280~16777216,314572800~25165824,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.438 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,205520896~16777216,230686720~8388608,247463936~25165824,293601280~16777216,314572800~25165824,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.443 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~25165824,205520896~20971520,230686720~12582912,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:09:57.443 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 18 diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.443 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~29360128,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.447 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~25165824,205520896~37748736,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:09:57.447 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to 19 diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:09:57.447 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:09:57.453 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:09:57.453 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 9 diff [8388608~12582912,54525952~8388608,79691776~8388608,100663296~8388608,130023424~8388608,180355072~8388608,398458880~12582912] 2024-03-29T05:09:57.453 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~12582912,54525952~8388608,79691776~8388608,100663296~8388608,130023424~8388608,180355072~8388608,398458880~12582912] 2024-03-29T05:09:57.454 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~16777216,54525952~8388608,79691776~12582912,100663296~12582912,130023424~12582912,180355072~12582912,398458880~16777216] 2024-03-29T05:09:57.454 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 10 diff [8388608~12582912,54525952~8388608,79691776~8388608,100663296~8388608,130023424~8388608,180355072~12582912,247463936~12582912,297795584~12582912,322961408~20971520,377487360~4194304,398458880~12582912] 2024-03-29T05:09:57.455 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~12582912,54525952~8388608,79691776~8388608,100663296~8388608,130023424~8388608,180355072~12582912,247463936~12582912,297795584~12582912,322961408~20971520,377487360~4194304,398458880~12582912] 2024-03-29T05:09:57.455 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~16777216,54525952~8388608,79691776~12582912,100663296~12582912,130023424~12582912,180355072~12582912,247463936~12582912,297795584~16777216,322961408~20971520,377487360~8388608,398458880~16777216] 2024-03-29T05:09:57.455 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 11 diff [8388608~12582912,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,155189248~8388608,180355072~12582912,213909504~4194304,247463936~12582912,297795584~12582912,322961408~16777216,364904448~4194304,377487360~4194304,398458880~12582912] 2024-03-29T05:09:57.455 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~12582912,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,155189248~8388608,180355072~12582912,213909504~4194304,247463936~12582912,297795584~12582912,322961408~16777216,364904448~4194304,377487360~4194304,398458880~12582912] 2024-03-29T05:09:57.456 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~16777216,46137344~16777216,71303168~20971520,100663296~12582912,121634816~20971520,155189248~8388608,180355072~16777216,213909504~8388608,247463936~12582912,297795584~16777216,322961408~20971520,364904448~8388608,377487360~8388608,398458880~16777216] 2024-03-29T05:09:57.456 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 12 diff [8388608~12582912,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,146800640~16777216,180355072~12582912,213909504~4194304,230686720~8388608,247463936~12582912,297795584~12582912,322961408~16777216,356515840~16777216,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.456 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~12582912,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,146800640~16777216,180355072~12582912,213909504~4194304,230686720~8388608,247463936~12582912,297795584~12582912,322961408~16777216,356515840~16777216,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.458 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~16777216,46137344~16777216,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,180355072~16777216,213909504~8388608,230686720~12582912,247463936~12582912,297795584~16777216,322961408~20971520,356515840~33554432,398458880~16777216] 2024-03-29T05:09:57.458 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 13 diff [8388608~16777216,29360128~4194304,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,146800640~16777216,171966464~20971520,213909504~4194304,230686720~8388608,247463936~12582912,264241152~8388608,297795584~12582912,322961408~16777216,356515840~16777216,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.458 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [8388608~16777216,29360128~4194304,46137344~16777216,71303168~16777216,100663296~8388608,121634816~16777216,146800640~16777216,171966464~20971520,213909504~4194304,230686720~8388608,247463936~12582912,264241152~8388608,297795584~12582912,322961408~16777216,356515840~16777216,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.461 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [8388608~16777216,29360128~4194304,46137344~16777216,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,171966464~25165824,213909504~8388608,230686720~12582912,247463936~12582912,264241152~8388608,297795584~16777216,322961408~20971520,356515840~33554432,398458880~16777216] 2024-03-29T05:09:57.461 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 14 diff [4194304~29360128,46137344~16777216,71303168~16777216,100663296~8388608,121634816~20971520,146800640~16777216,171966464~20971520,205520896~4194304,213909504~4194304,230686720~8388608,247463936~12582912,264241152~8388608,297795584~12582912,318767104~20971520,343932928~4194304,356515840~16777216,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.461 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~29360128,46137344~16777216,71303168~16777216,100663296~8388608,121634816~20971520,146800640~16777216,171966464~20971520,205520896~4194304,213909504~4194304,230686720~8388608,247463936~12582912,264241152~8388608,297795584~12582912,318767104~20971520,343932928~4194304,356515840~16777216,377487360~8388608,398458880~12582912] 2024-03-29T05:09:57.464 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~33554432,46137344~16777216,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,171966464~25165824,205520896~4194304,213909504~8388608,230686720~12582912,247463936~12582912,264241152~8388608,297795584~16777216,318767104~29360128,356515840~33554432,398458880~16777216] 2024-03-29T05:09:57.464 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 15 diff [4194304~29360128,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~4194304,213909504~8388608,230686720~8388608,247463936~25165824,297795584~12582912,314572800~25165824,343932928~4194304,356515840~16777216,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.464 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~29360128,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~4194304,213909504~8388608,230686720~8388608,247463936~25165824,297795584~12582912,314572800~25165824,343932928~4194304,356515840~16777216,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.467 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~33554432,46137344~16777216,67108864~25165824,100663296~41943040,146800640~16777216,171966464~25165824,205520896~4194304,213909504~12582912,230686720~12582912,247463936~29360128,297795584~50331648,356515840~33554432,398458880~16777216] 2024-03-29T05:09:57.467 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 16 diff [4194304~37748736,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~16777216,230686720~8388608,247463936~25165824,297795584~12582912,314572800~25165824,343932928~4194304,356515840~16777216,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.467 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,100663296~41943040,146800640~16777216,171966464~20971520,205520896~16777216,230686720~8388608,247463936~25165824,297795584~12582912,314572800~25165824,343932928~4194304,356515840~16777216,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.470 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~25165824,100663296~41943040,146800640~16777216,171966464~25165824,205520896~20971520,230686720~12582912,247463936~29360128,297795584~50331648,356515840~33554432,398458880~16777216] 2024-03-29T05:09:57.470 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 17 diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,205520896~16777216,230686720~8388608,247463936~25165824,293601280~16777216,314572800~25165824,343932928~4194304,356515840~16777216,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.470 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,205520896~16777216,230686720~8388608,247463936~25165824,293601280~16777216,314572800~25165824,343932928~4194304,356515840~16777216,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.474 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~25165824,205520896~20971520,230686720~12582912,247463936~29360128,293601280~54525952,356515840~33554432,398458880~16777216] 2024-03-29T05:09:57.475 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 18 diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~16777216,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.475 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~16777216,377487360~12582912,398458880~12582912] 2024-03-29T05:09:57.478 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~25165824,205520896~37748736,247463936~29360128,293601280~58720256,356515840~33554432,398458880~16777216] 2024-03-29T05:09:57.478 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to 19 diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.478 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.486 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~16777216] 2024-03-29T05:09:57.486 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 10 diff [180355072~8388608,247463936~12582912,297795584~12582912,322961408~20971520,377487360~4194304] 2024-03-29T05:09:57.486 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [180355072~8388608,247463936~12582912,297795584~12582912,322961408~20971520,377487360~4194304] 2024-03-29T05:09:57.486 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [180355072~8388608,247463936~12582912,297795584~16777216,322961408~20971520,377487360~8388608] 2024-03-29T05:09:57.486 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 11 diff [16777216~8388608,46137344~8388608,71303168~8388608,121634816~8388608,155189248~8388608,180355072~12582912,213909504~4194304,247463936~12582912,297795584~12582912,322961408~16777216,364904448~4194304,377487360~4194304] 2024-03-29T05:09:57.486 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [16777216~8388608,46137344~8388608,71303168~8388608,121634816~8388608,155189248~8388608,180355072~12582912,213909504~4194304,247463936~12582912,297795584~12582912,322961408~16777216,364904448~4194304,377487360~4194304] 2024-03-29T05:09:57.489 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [16777216~8388608,46137344~12582912,71303168~12582912,121634816~8388608,155189248~8388608,180355072~16777216,213909504~8388608,247463936~12582912,297795584~16777216,322961408~20971520,364904448~8388608,377487360~8388608] 2024-03-29T05:09:57.489 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 12 diff [16777216~8388608,46137344~8388608,71303168~12582912,100663296~8388608,121634816~8388608,146800640~16777216,180355072~12582912,213909504~4194304,230686720~8388608,247463936~12582912,297795584~12582912,322961408~16777216,356515840~16777216,377487360~8388608,398458880~8388608] 2024-03-29T05:09:57.490 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [16777216~8388608,46137344~8388608,71303168~12582912,100663296~8388608,121634816~8388608,146800640~16777216,180355072~12582912,213909504~4194304,230686720~8388608,247463936~12582912,297795584~12582912,322961408~16777216,356515840~16777216,377487360~8388608,398458880~8388608] 2024-03-29T05:09:57.490 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [16777216~8388608,46137344~12582912,71303168~16777216,100663296~12582912,121634816~8388608,146800640~16777216,180355072~16777216,213909504~8388608,230686720~12582912,247463936~12582912,297795584~16777216,322961408~20971520,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.490 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 13 diff [16777216~8388608,29360128~4194304,46137344~8388608,71303168~12582912,88080384~4194304,100663296~8388608,121634816~8388608,146800640~16777216,171966464~20971520,213909504~4194304,230686720~8388608,247463936~12582912,264241152~8388608,297795584~12582912,322961408~16777216,356515840~16777216,377487360~8388608,398458880~8388608] 2024-03-29T05:09:57.490 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [16777216~8388608,29360128~4194304,46137344~8388608,71303168~12582912,88080384~4194304,100663296~8388608,121634816~8388608,146800640~16777216,171966464~20971520,213909504~4194304,230686720~8388608,247463936~12582912,264241152~8388608,297795584~12582912,322961408~16777216,356515840~16777216,377487360~8388608,398458880~8388608] 2024-03-29T05:09:57.494 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [16777216~8388608,29360128~4194304,46137344~12582912,71303168~20971520,100663296~12582912,121634816~8388608,146800640~16777216,171966464~25165824,213909504~8388608,230686720~12582912,247463936~12582912,264241152~8388608,297795584~16777216,322961408~20971520,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.495 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 14 diff [4194304~4194304,16777216~16777216,46137344~8388608,71303168~12582912,88080384~4194304,100663296~8388608,121634816~16777216,146800640~16777216,171966464~20971520,205520896~4194304,213909504~4194304,230686720~8388608,247463936~12582912,264241152~8388608,297795584~12582912,318767104~20971520,343932928~4194304,356515840~16777216,377487360~8388608,398458880~8388608] 2024-03-29T05:09:57.495 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~16777216,46137344~8388608,71303168~12582912,88080384~4194304,100663296~8388608,121634816~16777216,146800640~16777216,171966464~20971520,205520896~4194304,213909504~4194304,230686720~8388608,247463936~12582912,264241152~8388608,297795584~12582912,318767104~20971520,343932928~4194304,356515840~16777216,377487360~8388608,398458880~8388608] 2024-03-29T05:09:57.496 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~20971520,46137344~12582912,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,171966464~25165824,205520896~4194304,213909504~8388608,230686720~12582912,247463936~12582912,264241152~8388608,297795584~16777216,318767104~29360128,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.496 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 15 diff [4194304~4194304,16777216~16777216,46137344~8388608,67108864~16777216,88080384~4194304,100663296~37748736,146800640~16777216,171966464~20971520,205520896~4194304,213909504~8388608,230686720~8388608,247463936~25165824,297795584~12582912,314572800~25165824,343932928~4194304,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.496 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~16777216,46137344~8388608,67108864~16777216,88080384~4194304,100663296~37748736,146800640~16777216,171966464~20971520,205520896~4194304,213909504~8388608,230686720~8388608,247463936~25165824,297795584~12582912,314572800~25165824,343932928~4194304,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.498 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~20971520,46137344~12582912,67108864~25165824,100663296~41943040,146800640~16777216,171966464~25165824,205520896~4194304,213909504~12582912,230686720~12582912,247463936~29360128,297795584~50331648,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.498 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 16 diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~4194304,100663296~37748736,146800640~16777216,171966464~20971520,205520896~16777216,230686720~8388608,247463936~25165824,297795584~12582912,314572800~25165824,343932928~4194304,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.498 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~4194304,100663296~37748736,146800640~16777216,171966464~20971520,205520896~16777216,230686720~8388608,247463936~25165824,297795584~12582912,314572800~25165824,343932928~4194304,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.499 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~25165824,100663296~41943040,146800640~16777216,171966464~25165824,205520896~20971520,230686720~12582912,247463936~29360128,297795584~50331648,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.499 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 17 diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,205520896~16777216,230686720~8388608,247463936~25165824,293601280~16777216,314572800~25165824,343932928~4194304,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.499 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,205520896~16777216,230686720~8388608,247463936~25165824,293601280~16777216,314572800~25165824,343932928~4194304,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.503 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~25165824,205520896~20971520,230686720~12582912,247463936~29360128,293601280~54525952,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.504 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 18 diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.504 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.508 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~25165824,205520896~37748736,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.508 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to 19 diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:09:57.509 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:09:57.512 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.512 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 11 diff [16777216~8388608,46137344~8388608,71303168~8388608,121634816~8388608,155189248~8388608,184549376~8388608,213909504~4194304,331350016~12582912,364904448~4194304] 2024-03-29T05:09:57.513 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [16777216~8388608,46137344~8388608,71303168~8388608,121634816~8388608,155189248~8388608,184549376~8388608,213909504~4194304,331350016~12582912,364904448~4194304] 2024-03-29T05:09:57.513 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [16777216~8388608,46137344~12582912,71303168~12582912,121634816~8388608,155189248~8388608,184549376~12582912,213909504~8388608,331350016~12582912,364904448~8388608] 2024-03-29T05:09:57.514 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 12 diff [16777216~8388608,46137344~8388608,71303168~12582912,100663296~8388608,121634816~8388608,146800640~16777216,184549376~8388608,213909504~4194304,230686720~8388608,331350016~12582912,356515840~16777216,381681664~4194304,398458880~8388608] 2024-03-29T05:09:57.514 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [16777216~8388608,46137344~8388608,71303168~12582912,100663296~8388608,121634816~8388608,146800640~16777216,184549376~8388608,213909504~4194304,230686720~8388608,331350016~12582912,356515840~16777216,381681664~4194304,398458880~8388608] 2024-03-29T05:09:57.516 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [16777216~8388608,46137344~12582912,71303168~16777216,100663296~12582912,121634816~8388608,146800640~16777216,184549376~12582912,213909504~8388608,230686720~12582912,331350016~12582912,356515840~20971520,381681664~8388608,398458880~12582912] 2024-03-29T05:09:57.516 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 13 diff [16777216~8388608,29360128~4194304,46137344~8388608,71303168~12582912,88080384~4194304,100663296~8388608,121634816~8388608,146800640~16777216,171966464~8388608,184549376~8388608,213909504~4194304,230686720~8388608,264241152~8388608,331350016~12582912,356515840~16777216,377487360~8388608,398458880~8388608] 2024-03-29T05:09:57.516 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [16777216~8388608,29360128~4194304,46137344~8388608,71303168~12582912,88080384~4194304,100663296~8388608,121634816~8388608,146800640~16777216,171966464~8388608,184549376~8388608,213909504~4194304,230686720~8388608,264241152~8388608,331350016~12582912,356515840~16777216,377487360~8388608,398458880~8388608] 2024-03-29T05:09:57.517 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [16777216~8388608,29360128~4194304,46137344~12582912,71303168~20971520,100663296~12582912,121634816~8388608,146800640~16777216,171966464~8388608,184549376~12582912,213909504~8388608,230686720~12582912,264241152~8388608,331350016~12582912,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.517 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 14 diff [4194304~4194304,16777216~16777216,46137344~8388608,71303168~12582912,88080384~4194304,100663296~8388608,121634816~16777216,146800640~16777216,171966464~8388608,184549376~8388608,205520896~4194304,213909504~4194304,230686720~8388608,247463936~4194304,264241152~8388608,318767104~8388608,331350016~16777216,356515840~16777216,377487360~8388608,398458880~8388608] 2024-03-29T05:09:57.517 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~16777216,46137344~8388608,71303168~12582912,88080384~4194304,100663296~8388608,121634816~16777216,146800640~16777216,171966464~8388608,184549376~8388608,205520896~4194304,213909504~4194304,230686720~8388608,247463936~4194304,264241152~8388608,318767104~8388608,331350016~16777216,356515840~16777216,377487360~8388608,398458880~8388608] 2024-03-29T05:09:57.520 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~20971520,46137344~12582912,71303168~20971520,100663296~12582912,121634816~20971520,146800640~16777216,171966464~8388608,184549376~12582912,205520896~4194304,213909504~8388608,230686720~12582912,247463936~8388608,264241152~8388608,318767104~8388608,331350016~16777216,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.520 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 15 diff [4194304~4194304,16777216~16777216,46137344~8388608,67108864~16777216,88080384~4194304,100663296~37748736,146800640~16777216,171966464~20971520,205520896~4194304,213909504~8388608,230686720~8388608,247463936~4194304,260046848~12582912,314572800~12582912,331350016~16777216,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.520 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~16777216,46137344~8388608,67108864~16777216,88080384~4194304,100663296~37748736,146800640~16777216,171966464~20971520,205520896~4194304,213909504~8388608,230686720~8388608,247463936~4194304,260046848~12582912,314572800~12582912,331350016~16777216,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.522 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~20971520,46137344~12582912,67108864~25165824,100663296~41943040,146800640~16777216,171966464~25165824,205520896~4194304,213909504~12582912,230686720~12582912,247463936~8388608,260046848~16777216,314572800~12582912,331350016~16777216,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.522 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 16 diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~4194304,100663296~37748736,146800640~16777216,171966464~20971520,205520896~16777216,230686720~8388608,247463936~25165824,306184192~4194304,314572800~12582912,331350016~16777216,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.522 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~4194304,100663296~37748736,146800640~16777216,171966464~20971520,205520896~16777216,230686720~8388608,247463936~25165824,306184192~4194304,314572800~12582912,331350016~16777216,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.525 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~25165824,100663296~41943040,146800640~16777216,171966464~25165824,205520896~20971520,230686720~12582912,247463936~29360128,306184192~20971520,331350016~16777216,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.525 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 17 diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,205520896~16777216,230686720~8388608,247463936~25165824,293601280~4194304,306184192~4194304,314572800~12582912,331350016~16777216,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.525 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,205520896~16777216,230686720~8388608,247463936~25165824,293601280~4194304,306184192~4194304,314572800~12582912,331350016~16777216,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.528 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~25165824,205520896~20971520,230686720~12582912,247463936~29360128,293601280~4194304,306184192~20971520,331350016~16777216,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.528 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 18 diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.528 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.532 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~25165824,205520896~37748736,247463936~29360128,293601280~33554432,331350016~20971520,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.532 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to 19 diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:09:57.532 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:09:57.536 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.536 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to 12 diff [71303168~4194304,79691776~8388608,100663296~8388608,146800640~8388608,184549376~8388608,230686720~8388608,356515840~8388608,369098752~4194304,381681664~4194304,398458880~8388608] 2024-03-29T05:09:57.536 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [71303168~4194304,79691776~8388608,100663296~8388608,146800640~8388608,184549376~8388608,230686720~8388608,356515840~8388608,369098752~4194304,381681664~4194304,398458880~8388608] 2024-03-29T05:09:57.538 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [71303168~16777216,100663296~12582912,146800640~12582912,184549376~8388608,230686720~12582912,356515840~8388608,369098752~8388608,381681664~8388608,398458880~12582912] 2024-03-29T05:09:57.538 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to 13 diff [16777216~8388608,29360128~4194304,46137344~8388608,71303168~4194304,79691776~12582912,100663296~8388608,146800640~8388608,171966464~8388608,184549376~8388608,230686720~8388608,264241152~8388608,356515840~16777216,377487360~8388608,398458880~8388608] 2024-03-29T05:09:57.538 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [16777216~8388608,29360128~4194304,46137344~8388608,71303168~4194304,79691776~12582912,100663296~8388608,146800640~8388608,171966464~8388608,184549376~8388608,230686720~8388608,264241152~8388608,356515840~16777216,377487360~8388608,398458880~8388608] 2024-03-29T05:09:57.540 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [16777216~8388608,29360128~4194304,46137344~8388608,71303168~20971520,100663296~12582912,146800640~12582912,171966464~8388608,184549376~8388608,230686720~12582912,264241152~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.540 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to 14 diff [4194304~4194304,16777216~16777216,46137344~8388608,71303168~4194304,79691776~12582912,100663296~8388608,121634816~4194304,130023424~8388608,146800640~8388608,171966464~8388608,184549376~8388608,205520896~4194304,230686720~8388608,247463936~4194304,264241152~8388608,318767104~8388608,331350016~4194304,343932928~4194304,356515840~16777216,377487360~8388608,398458880~8388608] 2024-03-29T05:09:57.540 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~16777216,46137344~8388608,71303168~4194304,79691776~12582912,100663296~8388608,121634816~4194304,130023424~8388608,146800640~8388608,171966464~8388608,184549376~8388608,205520896~4194304,230686720~8388608,247463936~4194304,264241152~8388608,318767104~8388608,331350016~4194304,343932928~4194304,356515840~16777216,377487360~8388608,398458880~8388608] 2024-03-29T05:09:57.543 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~20971520,46137344~8388608,71303168~20971520,100663296~12582912,121634816~4194304,130023424~12582912,146800640~12582912,171966464~8388608,184549376~8388608,205520896~4194304,230686720~12582912,247463936~8388608,264241152~8388608,318767104~8388608,331350016~8388608,343932928~4194304,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.543 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to 15 diff [4194304~4194304,16777216~16777216,46137344~8388608,67108864~8388608,79691776~12582912,100663296~25165824,130023424~8388608,146800640~8388608,171966464~20971520,205520896~4194304,218103808~8388608,230686720~8388608,247463936~4194304,260046848~12582912,314572800~12582912,331350016~4194304,343932928~4194304,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.543 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~16777216,46137344~8388608,67108864~8388608,79691776~12582912,100663296~25165824,130023424~8388608,146800640~8388608,171966464~20971520,205520896~4194304,218103808~8388608,230686720~8388608,247463936~4194304,260046848~12582912,314572800~12582912,331350016~4194304,343932928~4194304,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.546 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~20971520,46137344~8388608,67108864~25165824,100663296~25165824,130023424~12582912,146800640~12582912,171966464~20971520,205520896~4194304,218103808~8388608,230686720~12582912,247463936~8388608,260046848~16777216,314572800~12582912,331350016~8388608,343932928~4194304,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.546 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to 16 diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~12582912,100663296~25165824,130023424~8388608,146800640~8388608,171966464~20971520,205520896~8388608,218103808~8388608,230686720~8388608,247463936~25165824,306184192~4194304,314572800~12582912,331350016~4194304,343932928~4194304,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.546 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~12582912,100663296~25165824,130023424~8388608,146800640~8388608,171966464~20971520,205520896~8388608,218103808~8388608,230686720~8388608,247463936~25165824,306184192~4194304,314572800~12582912,331350016~4194304,343932928~4194304,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.548 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~25165824,100663296~25165824,130023424~12582912,146800640~12582912,171966464~20971520,205520896~8388608,218103808~8388608,230686720~12582912,247463936~29360128,306184192~20971520,331350016~8388608,343932928~4194304,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.549 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to 17 diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~25165824,130023424~8388608,142606336~12582912,171966464~20971520,205520896~8388608,218103808~8388608,230686720~8388608,247463936~25165824,293601280~4194304,306184192~4194304,314572800~12582912,331350016~4194304,343932928~4194304,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.549 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~25165824,130023424~8388608,142606336~12582912,171966464~20971520,205520896~8388608,218103808~8388608,230686720~8388608,247463936~25165824,293601280~4194304,306184192~4194304,314572800~12582912,331350016~4194304,343932928~4194304,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.551 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~29360128,100663296~25165824,130023424~29360128,171966464~20971520,205520896~20971520,230686720~12582912,247463936~29360128,293601280~4194304,306184192~20971520,331350016~8388608,343932928~4194304,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.551 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to 18 diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.551 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~16777216,377487360~12582912,398458880~8388608] 2024-03-29T05:09:57.553 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~29360128,100663296~58720256,171966464~20971520,205520896~37748736,247463936~29360128,293601280~33554432,331350016~8388608,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.553 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to 19 diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:09:57.553 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:09:57.558 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~29360128,100663296~58720256,171966464~20971520,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.558 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to 13 diff [16777216~8388608,29360128~4194304,46137344~8388608,88080384~4194304,104857600~4194304,171966464~8388608,264241152~8388608,364904448~8388608,377487360~8388608] 2024-03-29T05:09:57.558 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [16777216~8388608,29360128~4194304,46137344~8388608,88080384~4194304,104857600~4194304,171966464~8388608,264241152~8388608,364904448~8388608,377487360~8388608] 2024-03-29T05:09:57.559 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [16777216~8388608,29360128~4194304,46137344~8388608,88080384~4194304,104857600~4194304,171966464~8388608,264241152~8388608,364904448~8388608,377487360~8388608] 2024-03-29T05:09:57.559 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to 14 diff [4194304~4194304,16777216~16777216,46137344~8388608,88080384~4194304,104857600~4194304,121634816~4194304,130023424~8388608,171966464~8388608,205520896~4194304,247463936~4194304,264241152~8388608,318767104~8388608,331350016~4194304,343932928~4194304,360710144~8388608,377487360~8388608] 2024-03-29T05:09:57.560 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~16777216,46137344~8388608,88080384~4194304,104857600~4194304,121634816~4194304,130023424~8388608,171966464~8388608,205520896~4194304,247463936~4194304,264241152~8388608,318767104~8388608,331350016~4194304,343932928~4194304,360710144~8388608,377487360~8388608] 2024-03-29T05:09:57.564 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~20971520,46137344~8388608,88080384~4194304,104857600~4194304,121634816~4194304,130023424~12582912,171966464~8388608,205520896~4194304,247463936~8388608,264241152~8388608,318767104~8388608,331350016~8388608,343932928~4194304,360710144~12582912,377487360~8388608] 2024-03-29T05:09:57.564 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to 15 diff [4194304~4194304,16777216~16777216,46137344~8388608,67108864~4194304,88080384~4194304,104857600~20971520,130023424~8388608,171966464~16777216,205520896~4194304,218103808~8388608,247463936~4194304,260046848~12582912,314572800~12582912,331350016~4194304,343932928~4194304,360710144~8388608,377487360~12582912] 2024-03-29T05:09:57.564 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~16777216,46137344~8388608,67108864~4194304,88080384~4194304,104857600~20971520,130023424~8388608,171966464~16777216,205520896~4194304,218103808~8388608,247463936~4194304,260046848~12582912,314572800~12582912,331350016~4194304,343932928~4194304,360710144~8388608,377487360~12582912] 2024-03-29T05:09:57.565 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~20971520,46137344~8388608,67108864~8388608,88080384~4194304,104857600~20971520,130023424~12582912,171966464~16777216,205520896~4194304,218103808~8388608,247463936~8388608,260046848~16777216,314572800~12582912,331350016~8388608,343932928~4194304,360710144~12582912,377487360~12582912] 2024-03-29T05:09:57.565 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to 16 diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~4194304,104857600~20971520,130023424~8388608,171966464~16777216,205520896~8388608,218103808~8388608,247463936~25165824,306184192~4194304,314572800~12582912,331350016~4194304,343932928~4194304,360710144~8388608,377487360~12582912] 2024-03-29T05:09:57.566 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~4194304,104857600~20971520,130023424~8388608,171966464~16777216,205520896~8388608,218103808~8388608,247463936~25165824,306184192~4194304,314572800~12582912,331350016~4194304,343932928~4194304,360710144~8388608,377487360~12582912] 2024-03-29T05:09:57.566 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~8388608,88080384~4194304,104857600~20971520,130023424~12582912,171966464~16777216,205520896~8388608,218103808~8388608,247463936~29360128,306184192~20971520,331350016~8388608,343932928~4194304,360710144~12582912,377487360~12582912] 2024-03-29T05:09:57.566 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to 17 diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,104857600~20971520,130023424~8388608,142606336~8388608,171966464~20971520,205520896~8388608,218103808~8388608,247463936~25165824,293601280~4194304,306184192~4194304,314572800~12582912,331350016~4194304,343932928~4194304,360710144~8388608,377487360~12582912] 2024-03-29T05:09:57.566 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,104857600~20971520,130023424~8388608,142606336~8388608,171966464~20971520,205520896~8388608,218103808~8388608,247463936~25165824,293601280~4194304,306184192~4194304,314572800~12582912,331350016~4194304,343932928~4194304,360710144~8388608,377487360~12582912] 2024-03-29T05:09:57.566 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~8388608,88080384~8388608,104857600~20971520,130023424~20971520,171966464~20971520,205520896~20971520,247463936~29360128,293601280~4194304,306184192~20971520,331350016~8388608,343932928~4194304,360710144~12582912,377487360~12582912] 2024-03-29T05:09:57.567 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to 18 diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,100663296~37748736,142606336~12582912,171966464~20971520,205520896~8388608,218103808~12582912,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~12582912,377487360~12582912] 2024-03-29T05:09:57.567 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,100663296~37748736,142606336~12582912,171966464~20971520,205520896~8388608,218103808~12582912,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~12582912,377487360~12582912] 2024-03-29T05:09:57.571 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~8388608,88080384~8388608,100663296~54525952,171966464~20971520,205520896~29360128,247463936~29360128,293601280~33554432,331350016~8388608,343932928~8388608,360710144~29360128] 2024-03-29T05:09:57.572 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to 19 diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:09:57.572 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:09:57.573 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~8388608,88080384~8388608,100663296~54525952,171966464~20971520,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,360710144~29360128] 2024-03-29T05:09:57.574 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to 14 diff [4194304~4194304,25165824~8388608,121634816~4194304,130023424~8388608,205520896~4194304,247463936~4194304,318767104~8388608,331350016~4194304,343932928~4194304,360710144~8388608] 2024-03-29T05:09:57.574 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,25165824~8388608,121634816~4194304,130023424~8388608,205520896~4194304,247463936~4194304,318767104~8388608,331350016~4194304,343932928~4194304,360710144~8388608] 2024-03-29T05:09:57.575 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,25165824~12582912,121634816~4194304,130023424~12582912,205520896~4194304,247463936~8388608,318767104~8388608,331350016~8388608,343932928~4194304,360710144~12582912] 2024-03-29T05:09:57.575 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to 15 diff [4194304~4194304,25165824~8388608,67108864~4194304,109051904~16777216,130023424~8388608,180355072~8388608,205520896~4194304,218103808~8388608,247463936~4194304,260046848~12582912,314572800~12582912,331350016~4194304,343932928~4194304,360710144~8388608,385875968~4194304] 2024-03-29T05:09:57.575 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,25165824~8388608,67108864~4194304,109051904~16777216,130023424~8388608,180355072~8388608,205520896~4194304,218103808~8388608,247463936~4194304,260046848~12582912,314572800~12582912,331350016~4194304,343932928~4194304,360710144~8388608,385875968~4194304] 2024-03-29T05:09:57.576 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,25165824~12582912,67108864~8388608,109051904~16777216,130023424~12582912,180355072~8388608,205520896~4194304,218103808~8388608,247463936~8388608,260046848~16777216,314572800~12582912,331350016~8388608,343932928~4194304,360710144~12582912,385875968~4194304] 2024-03-29T05:09:57.576 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to 16 diff [4194304~4194304,25165824~16777216,67108864~4194304,109051904~16777216,130023424~8388608,180355072~8388608,205520896~8388608,218103808~8388608,247463936~25165824,306184192~4194304,314572800~12582912,331350016~4194304,343932928~4194304,360710144~8388608,385875968~4194304] 2024-03-29T05:09:57.577 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,25165824~16777216,67108864~4194304,109051904~16777216,130023424~8388608,180355072~8388608,205520896~8388608,218103808~8388608,247463936~25165824,306184192~4194304,314572800~12582912,331350016~4194304,343932928~4194304,360710144~8388608,385875968~4194304] 2024-03-29T05:09:57.578 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,25165824~20971520,67108864~8388608,109051904~16777216,130023424~12582912,180355072~8388608,205520896~8388608,218103808~8388608,247463936~29360128,306184192~20971520,331350016~8388608,343932928~4194304,360710144~12582912,385875968~4194304] 2024-03-29T05:09:57.578 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to 17 diff [4194304~4194304,25165824~16777216,67108864~4194304,92274688~4194304,109051904~16777216,130023424~8388608,142606336~8388608,180355072~12582912,205520896~8388608,218103808~8388608,247463936~25165824,293601280~4194304,306184192~4194304,314572800~12582912,331350016~4194304,343932928~4194304,360710144~8388608,385875968~4194304] 2024-03-29T05:09:57.578 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,25165824~16777216,67108864~4194304,92274688~4194304,109051904~16777216,130023424~8388608,142606336~8388608,180355072~12582912,205520896~8388608,218103808~8388608,247463936~25165824,293601280~4194304,306184192~4194304,314572800~12582912,331350016~4194304,343932928~4194304,360710144~8388608,385875968~4194304] 2024-03-29T05:09:57.585 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,25165824~20971520,67108864~8388608,92274688~4194304,109051904~16777216,130023424~20971520,180355072~12582912,205520896~20971520,247463936~29360128,293601280~4194304,306184192~20971520,331350016~8388608,343932928~4194304,360710144~12582912,385875968~4194304] 2024-03-29T05:09:57.585 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to 18 diff [4194304~4194304,25165824~16777216,67108864~4194304,92274688~4194304,100663296~37748736,142606336~12582912,180355072~12582912,205520896~8388608,218103808~12582912,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~12582912,385875968~4194304] 2024-03-29T05:09:57.585 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,25165824~16777216,67108864~4194304,92274688~4194304,100663296~37748736,142606336~12582912,180355072~12582912,205520896~8388608,218103808~12582912,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~12582912,385875968~4194304] 2024-03-29T05:09:57.586 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,25165824~20971520,67108864~8388608,92274688~4194304,100663296~54525952,180355072~12582912,205520896~29360128,247463936~29360128,293601280~33554432,331350016~8388608,343932928~8388608,360710144~16777216,385875968~4194304] 2024-03-29T05:09:57.586 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to 19 diff [4194304~4194304,25165824~16777216,67108864~4194304,92274688~4194304,100663296~37748736,142606336~12582912,180355072~12582912,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:09:57.586 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,25165824~16777216,67108864~4194304,92274688~4194304,100663296~37748736,142606336~12582912,180355072~12582912,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:09:57.587 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,25165824~20971520,67108864~8388608,92274688~4194304,100663296~54525952,180355072~12582912,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:09:57.587 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to 15 diff [25165824~4194304,67108864~4194304,109051904~16777216,180355072~8388608,218103808~8388608,260046848~12582912,314572800~8388608,385875968~4194304] 2024-03-29T05:09:57.587 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~4194304,67108864~4194304,109051904~16777216,180355072~8388608,218103808~8388608,260046848~12582912,314572800~8388608,385875968~4194304] 2024-03-29T05:09:57.587 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~4194304,67108864~8388608,109051904~16777216,180355072~8388608,218103808~8388608,260046848~16777216,314572800~8388608,385875968~4194304] 2024-03-29T05:09:57.587 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to 16 diff [25165824~4194304,33554432~8388608,67108864~4194304,109051904~16777216,180355072~8388608,209715200~4194304,218103808~8388608,251658240~20971520,306184192~4194304,314572800~8388608,331350016~4194304,385875968~4194304] 2024-03-29T05:09:57.587 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~4194304,33554432~8388608,67108864~4194304,109051904~16777216,180355072~8388608,209715200~4194304,218103808~8388608,251658240~20971520,306184192~4194304,314572800~8388608,331350016~4194304,385875968~4194304] 2024-03-29T05:09:57.588 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~20971520,67108864~8388608,109051904~16777216,180355072~8388608,209715200~4194304,218103808~8388608,251658240~25165824,306184192~16777216,331350016~4194304,385875968~4194304] 2024-03-29T05:09:57.588 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to 17 diff [25165824~4194304,33554432~8388608,67108864~4194304,92274688~4194304,109051904~16777216,138412032~12582912,180355072~12582912,209715200~4194304,218103808~8388608,251658240~20971520,293601280~4194304,306184192~4194304,314572800~8388608,331350016~4194304,385875968~4194304] 2024-03-29T05:09:57.588 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~4194304,33554432~8388608,67108864~4194304,92274688~4194304,109051904~16777216,138412032~12582912,180355072~12582912,209715200~4194304,218103808~8388608,251658240~20971520,293601280~4194304,306184192~4194304,314572800~8388608,331350016~4194304,385875968~4194304] 2024-03-29T05:09:57.590 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~20971520,67108864~8388608,92274688~4194304,109051904~16777216,138412032~12582912,180355072~12582912,209715200~16777216,251658240~25165824,293601280~4194304,306184192~16777216,331350016~4194304,385875968~4194304] 2024-03-29T05:09:57.591 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to 18 diff [25165824~4194304,33554432~8388608,67108864~4194304,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,209715200~4194304,218103808~12582912,251658240~20971520,293601280~8388608,306184192~4194304,314572800~8388608,331350016~4194304,348127232~4194304,369098752~4194304,385875968~4194304] 2024-03-29T05:09:57.591 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~4194304,33554432~8388608,67108864~4194304,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,209715200~4194304,218103808~12582912,251658240~20971520,293601280~8388608,306184192~4194304,314572800~8388608,331350016~4194304,348127232~4194304,369098752~4194304,385875968~4194304] 2024-03-29T05:09:57.593 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~20971520,67108864~8388608,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,209715200~25165824,251658240~25165824,293601280~29360128,331350016~4194304,348127232~4194304,369098752~8388608,385875968~4194304] 2024-03-29T05:09:57.593 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to 19 diff [25165824~16777216,67108864~4194304,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,218103808~12582912,234881024~4194304,251658240~20971520,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,348127232~4194304,369098752~12582912,385875968~4194304] 2024-03-29T05:09:57.593 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~16777216,67108864~4194304,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,218103808~12582912,234881024~4194304,251658240~20971520,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,348127232~4194304,369098752~12582912,385875968~4194304] 2024-03-29T05:09:57.595 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~20971520,67108864~8388608,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,197132288~8388608,209715200~33554432,251658240~25165824,293601280~41943040,348127232~4194304,369098752~12582912,385875968~4194304] 2024-03-29T05:09:57.596 INFO:tasks.workunit.client.0.smithi181.stdout:from 15 to 16 diff [25165824~4194304,33554432~8388608,209715200~4194304,251658240~12582912,306184192~4194304,331350016~4194304] 2024-03-29T05:09:57.596 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~4194304,33554432~8388608,209715200~4194304,251658240~12582912,306184192~4194304,331350016~4194304] 2024-03-29T05:09:57.597 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~20971520,209715200~4194304,251658240~16777216,306184192~8388608,331350016~4194304] 2024-03-29T05:09:57.597 INFO:tasks.workunit.client.0.smithi181.stdout:from 15 to 17 diff [25165824~4194304,33554432~8388608,92274688~4194304,138412032~12582912,180355072~12582912,209715200~4194304,251658240~12582912,293601280~4194304,306184192~4194304,314572800~4194304,331350016~4194304] 2024-03-29T05:09:57.597 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~4194304,33554432~8388608,92274688~4194304,138412032~12582912,180355072~12582912,209715200~4194304,251658240~12582912,293601280~4194304,306184192~4194304,314572800~4194304,331350016~4194304] 2024-03-29T05:09:57.599 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~20971520,92274688~4194304,138412032~12582912,180355072~12582912,209715200~8388608,251658240~16777216,293601280~4194304,306184192~16777216,331350016~4194304] 2024-03-29T05:09:57.599 INFO:tasks.workunit.client.0.smithi181.stdout:from 15 to 18 diff [25165824~4194304,33554432~8388608,92274688~4194304,100663296~8388608,125829120~8388608,138412032~16777216,180355072~12582912,209715200~4194304,222298112~8388608,251658240~12582912,293601280~8388608,306184192~4194304,314572800~4194304,331350016~4194304,348127232~4194304,369098752~4194304] 2024-03-29T05:09:57.599 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~4194304,33554432~8388608,92274688~4194304,100663296~8388608,125829120~8388608,138412032~16777216,180355072~12582912,209715200~4194304,222298112~8388608,251658240~12582912,293601280~8388608,306184192~4194304,314572800~4194304,331350016~4194304,348127232~4194304,369098752~4194304] 2024-03-29T05:09:57.601 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~20971520,92274688~4194304,100663296~12582912,125829120~8388608,138412032~16777216,180355072~12582912,209715200~8388608,222298112~12582912,251658240~16777216,293601280~29360128,331350016~4194304,348127232~4194304,369098752~8388608] 2024-03-29T05:09:57.601 INFO:tasks.workunit.client.0.smithi181.stdout:from 15 to 19 diff [25165824~16777216,92274688~4194304,100663296~8388608,117440512~8388608,130023424~4194304,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,222298112~8388608,234881024~4194304,251658240~12582912,293601280~8388608,306184192~4194304,314572800~4194304,322961408~4194304,331350016~4194304,348127232~4194304,369098752~12582912] 2024-03-29T05:09:57.601 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~16777216,92274688~4194304,100663296~8388608,117440512~8388608,130023424~4194304,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,222298112~8388608,234881024~4194304,251658240~12582912,293601280~8388608,306184192~4194304,314572800~4194304,322961408~4194304,331350016~4194304,348127232~4194304,369098752~12582912] 2024-03-29T05:09:57.603 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~20971520,92274688~4194304,100663296~12582912,117440512~16777216,138412032~16777216,180355072~12582912,197132288~8388608,209715200~8388608,222298112~20971520,251658240~16777216,293601280~41943040,348127232~4194304,369098752~12582912] 2024-03-29T05:09:57.603 INFO:tasks.workunit.client.0.smithi181.stdout:from 16 to 17 diff [92274688~4194304,138412032~12582912,180355072~12582912,209715200~4194304,293601280~4194304,314572800~4194304] 2024-03-29T05:09:57.604 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [92274688~4194304,138412032~12582912,180355072~12582912,209715200~4194304,293601280~4194304,314572800~4194304] 2024-03-29T05:09:57.605 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [92274688~4194304,138412032~12582912,180355072~12582912,209715200~8388608,293601280~4194304,314572800~8388608] 2024-03-29T05:09:57.605 INFO:tasks.workunit.client.0.smithi181.stdout:from 16 to 18 diff [92274688~4194304,100663296~8388608,125829120~8388608,138412032~16777216,180355072~12582912,209715200~4194304,222298112~8388608,260046848~4194304,293601280~8388608,314572800~4194304,348127232~4194304,369098752~4194304] 2024-03-29T05:09:57.605 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [92274688~4194304,100663296~8388608,125829120~8388608,138412032~16777216,180355072~12582912,209715200~4194304,222298112~8388608,260046848~4194304,293601280~8388608,314572800~4194304,348127232~4194304,369098752~4194304] 2024-03-29T05:09:57.607 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [92274688~4194304,100663296~12582912,125829120~8388608,138412032~16777216,180355072~12582912,209715200~8388608,222298112~12582912,260046848~4194304,293601280~12582912,314572800~8388608,348127232~4194304,369098752~8388608] 2024-03-29T05:09:57.607 INFO:tasks.workunit.client.0.smithi181.stdout:from 16 to 19 diff [25165824~8388608,92274688~4194304,100663296~8388608,117440512~8388608,130023424~4194304,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,222298112~8388608,234881024~4194304,255852544~8388608,293601280~8388608,314572800~4194304,322961408~4194304,348127232~4194304,369098752~12582912] 2024-03-29T05:09:57.607 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~8388608,92274688~4194304,100663296~8388608,117440512~8388608,130023424~4194304,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,222298112~8388608,234881024~4194304,255852544~8388608,293601280~8388608,314572800~4194304,322961408~4194304,348127232~4194304,369098752~12582912] 2024-03-29T05:09:57.610 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~12582912,92274688~4194304,100663296~12582912,117440512~16777216,138412032~16777216,180355072~12582912,197132288~8388608,209715200~8388608,222298112~20971520,255852544~8388608,293601280~12582912,314572800~16777216,348127232~4194304,369098752~12582912] 2024-03-29T05:09:57.610 INFO:tasks.workunit.client.0.smithi181.stdout:from 17 to 18 diff [100663296~8388608,125829120~8388608,142606336~8388608,222298112~8388608,260046848~4194304,293601280~8388608,318767104~4194304,348127232~4194304,369098752~4194304] 2024-03-29T05:09:57.610 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [100663296~8388608,125829120~8388608,142606336~8388608,222298112~8388608,260046848~4194304,293601280~8388608,318767104~4194304,348127232~4194304,369098752~4194304] 2024-03-29T05:09:57.610 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [100663296~12582912,125829120~8388608,142606336~12582912,222298112~12582912,260046848~4194304,293601280~12582912,318767104~4194304,348127232~4194304,369098752~8388608] 2024-03-29T05:09:57.611 INFO:tasks.workunit.client.0.smithi181.stdout:from 17 to 19 diff [25165824~8388608,100663296~8388608,117440512~8388608,130023424~4194304,142606336~8388608,188743680~4194304,197132288~4194304,222298112~8388608,234881024~4194304,255852544~8388608,293601280~8388608,318767104~8388608,348127232~4194304,369098752~12582912] 2024-03-29T05:09:57.611 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~8388608,100663296~8388608,117440512~8388608,130023424~4194304,142606336~8388608,188743680~4194304,197132288~4194304,222298112~8388608,234881024~4194304,255852544~8388608,293601280~8388608,318767104~8388608,348127232~4194304,369098752~12582912] 2024-03-29T05:09:57.613 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~12582912,100663296~12582912,117440512~16777216,142606336~12582912,188743680~4194304,197132288~8388608,222298112~20971520,255852544~8388608,293601280~12582912,318767104~12582912,348127232~4194304,369098752~12582912] 2024-03-29T05:09:57.613 INFO:tasks.workunit.client.0.smithi181.stdout:from 18 to 19 diff [25165824~8388608,100663296~4194304,117440512~8388608,142606336~4194304,188743680~4194304,197132288~4194304,234881024~4194304,255852544~8388608,322961408~4194304,373293056~8388608] 2024-03-29T05:09:57.613 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~8388608,100663296~4194304,117440512~8388608,142606336~4194304,188743680~4194304,197132288~4194304,234881024~4194304,255852544~8388608,322961408~4194304,373293056~8388608] 2024-03-29T05:09:57.614 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~12582912,100663296~8388608,117440512~12582912,142606336~4194304,188743680~4194304,197132288~8388608,234881024~8388608,255852544~8388608,322961408~8388608,373293056~8388608] 2024-03-29T05:09:57.918 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:09:57.918 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:09:57.939 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:09:57.939 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:09:57.939 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:09:57.946 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:09:57.946 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:09:57.946 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:09:57.952 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:09:57.953 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:09:57.953 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:09:57.958 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:09:57.958 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:09:57.958 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:09:57.965 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~109051904,285212672~130023424] 2024-03-29T05:09:57.965 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:09:57.965 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:09:57.970 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:09:57.970 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:09:57.970 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:09:57.975 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:09:57.975 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:09:57.975 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:09:57.980 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:09:57.980 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.980 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.985 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~16777216] 2024-03-29T05:09:57.985 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:09:57.985 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:09:57.988 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.988 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:09:57.988 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:09:57.992 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.992 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:09:57.992 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:09:57.996 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~29360128,100663296~58720256,171966464~20971520,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:09:57.996 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:09:57.996 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:09:57.998 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~8388608,88080384~8388608,100663296~54525952,171966464~20971520,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,360710144~29360128] 2024-03-29T05:09:57.999 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to HEAD diff [4194304~4194304,25165824~16777216,67108864~4194304,92274688~4194304,100663296~37748736,142606336~12582912,180355072~12582912,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:09:57.999 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,25165824~16777216,67108864~4194304,92274688~4194304,100663296~37748736,142606336~12582912,180355072~12582912,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:09:58.001 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,25165824~20971520,67108864~8388608,92274688~4194304,100663296~54525952,180355072~12582912,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:09:58.001 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to HEAD diff [25165824~16777216,67108864~4194304,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,218103808~12582912,234881024~4194304,251658240~20971520,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,348127232~4194304,369098752~12582912,385875968~4194304] 2024-03-29T05:09:58.001 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~16777216,67108864~4194304,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,218103808~12582912,234881024~4194304,251658240~20971520,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,348127232~4194304,369098752~12582912,385875968~4194304] 2024-03-29T05:09:58.003 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~20971520,67108864~8388608,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,197132288~8388608,209715200~33554432,251658240~25165824,293601280~41943040,348127232~4194304,369098752~12582912,385875968~4194304] 2024-03-29T05:09:58.004 INFO:tasks.workunit.client.0.smithi181.stdout:from 15 to HEAD diff [25165824~16777216,92274688~4194304,100663296~8388608,117440512~8388608,130023424~4194304,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,222298112~8388608,234881024~4194304,251658240~12582912,293601280~8388608,306184192~4194304,314572800~4194304,322961408~4194304,331350016~4194304,348127232~4194304,369098752~12582912] 2024-03-29T05:09:58.004 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~16777216,92274688~4194304,100663296~8388608,117440512~8388608,130023424~4194304,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,222298112~8388608,234881024~4194304,251658240~12582912,293601280~8388608,306184192~4194304,314572800~4194304,322961408~4194304,331350016~4194304,348127232~4194304,369098752~12582912] 2024-03-29T05:09:58.006 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~20971520,92274688~4194304,100663296~12582912,117440512~16777216,138412032~16777216,180355072~12582912,197132288~8388608,209715200~8388608,222298112~20971520,251658240~16777216,293601280~41943040,348127232~4194304,369098752~12582912] 2024-03-29T05:09:58.006 INFO:tasks.workunit.client.0.smithi181.stdout:from 16 to HEAD diff [25165824~8388608,92274688~4194304,100663296~8388608,117440512~8388608,130023424~4194304,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,222298112~8388608,234881024~4194304,255852544~8388608,293601280~8388608,314572800~4194304,322961408~4194304,348127232~4194304,369098752~12582912] 2024-03-29T05:09:58.006 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~8388608,92274688~4194304,100663296~8388608,117440512~8388608,130023424~4194304,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,222298112~8388608,234881024~4194304,255852544~8388608,293601280~8388608,314572800~4194304,322961408~4194304,348127232~4194304,369098752~12582912] 2024-03-29T05:09:58.008 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~12582912,92274688~4194304,100663296~12582912,117440512~16777216,138412032~16777216,180355072~12582912,197132288~8388608,209715200~8388608,222298112~20971520,255852544~8388608,293601280~12582912,314572800~16777216,348127232~4194304,369098752~12582912] 2024-03-29T05:09:58.008 INFO:tasks.workunit.client.0.smithi181.stdout:from 17 to HEAD diff [25165824~8388608,100663296~8388608,117440512~8388608,130023424~4194304,142606336~8388608,188743680~4194304,197132288~4194304,222298112~8388608,234881024~4194304,255852544~8388608,293601280~8388608,318767104~8388608,348127232~4194304,369098752~12582912] 2024-03-29T05:09:58.008 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~8388608,100663296~8388608,117440512~8388608,130023424~4194304,142606336~8388608,188743680~4194304,197132288~4194304,222298112~8388608,234881024~4194304,255852544~8388608,293601280~8388608,318767104~8388608,348127232~4194304,369098752~12582912] 2024-03-29T05:09:58.008 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~12582912,100663296~12582912,117440512~16777216,142606336~12582912,188743680~4194304,197132288~8388608,222298112~20971520,255852544~8388608,293601280~12582912,318767104~12582912,348127232~4194304,369098752~12582912] 2024-03-29T05:09:58.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:57 smithi181 ceph-mon[106466]: failed to encode map e947 with expected crc 2024-03-29T05:09:58.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:57 smithi181 ceph-mon[106466]: osdmap e947: 8 total, 8 up, 8 in 2024-03-29T05:09:58.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:57 smithi181 ceph-mon[106466]: failed to encode map e947 with expected crc 2024-03-29T05:09:58.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:57 smithi181 ceph-mon[101970]: failed to encode map e947 with expected crc 2024-03-29T05:09:58.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:57 smithi181 ceph-mon[101970]: osdmap e947: 8 total, 8 up, 8 in 2024-03-29T05:09:58.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:57 smithi181 ceph-mon[101970]: failed to encode map e947 with expected crc 2024-03-29T05:09:58.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:57 smithi187 ceph-mon[93536]: failed to encode map e947 with expected crc 2024-03-29T05:09:58.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:57 smithi187 ceph-mon[93536]: osdmap e947: 8 total, 8 up, 8 in 2024-03-29T05:09:58.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:57 smithi187 ceph-mon[93536]: failed to encode map e947 with expected crc 2024-03-29T05:09:58.927 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:09:58.927 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:09:58.949 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:09:58.949 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:09:58.949 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:09:58.956 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:09:58.956 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:09:58.956 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:09:58.963 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:09:58.963 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:09:58.964 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:09:58.970 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:09:58.970 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:09:58.970 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:09:58.977 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~109051904,285212672~130023424] 2024-03-29T05:09:58.977 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:09:58.978 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:09:58.983 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:09:58.983 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:09:58.983 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:09:58.988 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:09:58.989 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:09:58.989 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:09:58.994 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:09:58.994 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:09:58.994 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:09:58.999 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~16777216] 2024-03-29T05:09:58.999 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:09:58.999 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:09:59.003 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:09:59.003 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:09:59.003 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:09:59.007 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:09:59.007 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:09:59.007 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:09:59.010 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~29360128,100663296~58720256,171966464~20971520,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:09:59.010 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:09:59.010 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:09:59.014 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~8388608,88080384~8388608,100663296~54525952,171966464~20971520,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,360710144~29360128] 2024-03-29T05:09:59.014 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to HEAD diff [4194304~4194304,25165824~16777216,67108864~4194304,92274688~4194304,100663296~37748736,142606336~12582912,180355072~12582912,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:09:59.014 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,25165824~16777216,67108864~4194304,92274688~4194304,100663296~37748736,142606336~12582912,180355072~12582912,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:09:59.017 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,25165824~20971520,67108864~8388608,92274688~4194304,100663296~54525952,180355072~12582912,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:09:59.017 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to HEAD diff [25165824~16777216,67108864~4194304,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,218103808~12582912,234881024~4194304,251658240~20971520,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,348127232~4194304,369098752~12582912,385875968~4194304] 2024-03-29T05:09:59.017 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~16777216,67108864~4194304,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,218103808~12582912,234881024~4194304,251658240~20971520,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,348127232~4194304,369098752~12582912,385875968~4194304] 2024-03-29T05:09:59.019 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~20971520,67108864~8388608,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,197132288~8388608,209715200~33554432,251658240~25165824,293601280~41943040,348127232~4194304,369098752~12582912,385875968~4194304] 2024-03-29T05:09:59.019 INFO:tasks.workunit.client.0.smithi181.stdout:from 15 to HEAD diff [25165824~16777216,92274688~4194304,100663296~8388608,117440512~8388608,130023424~4194304,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,222298112~8388608,234881024~4194304,251658240~12582912,293601280~8388608,306184192~4194304,314572800~4194304,322961408~4194304,331350016~4194304,348127232~4194304,369098752~12582912] 2024-03-29T05:09:59.019 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~16777216,92274688~4194304,100663296~8388608,117440512~8388608,130023424~4194304,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,222298112~8388608,234881024~4194304,251658240~12582912,293601280~8388608,306184192~4194304,314572800~4194304,322961408~4194304,331350016~4194304,348127232~4194304,369098752~12582912] 2024-03-29T05:09:59.021 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~20971520,92274688~4194304,100663296~12582912,117440512~16777216,138412032~16777216,180355072~12582912,197132288~8388608,209715200~8388608,222298112~20971520,251658240~16777216,293601280~41943040,348127232~4194304,369098752~12582912] 2024-03-29T05:09:59.021 INFO:tasks.workunit.client.0.smithi181.stdout:from 16 to HEAD diff [25165824~8388608,92274688~4194304,100663296~8388608,117440512~8388608,130023424~4194304,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,222298112~8388608,234881024~4194304,255852544~8388608,293601280~8388608,314572800~4194304,322961408~4194304,348127232~4194304,369098752~12582912] 2024-03-29T05:09:59.021 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~8388608,92274688~4194304,100663296~8388608,117440512~8388608,130023424~4194304,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,222298112~8388608,234881024~4194304,255852544~8388608,293601280~8388608,314572800~4194304,322961408~4194304,348127232~4194304,369098752~12582912] 2024-03-29T05:09:59.022 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~12582912,92274688~4194304,100663296~12582912,117440512~16777216,138412032~16777216,180355072~12582912,197132288~8388608,209715200~8388608,222298112~20971520,255852544~8388608,293601280~12582912,314572800~16777216,348127232~4194304,369098752~12582912] 2024-03-29T05:09:59.203 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:09:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:09:58] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.43.0" 2024-03-29T05:09:59.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:58 smithi181 ceph-mon[101970]: pgmap v1151: 105 pgs: 105 active+clean; 807 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 116 KiB/s rd, 33 MiB/s wr, 169 op/s 2024-03-29T05:09:59.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:58 smithi181 ceph-mon[101970]: osdmap e948: 8 total, 8 up, 8 in 2024-03-29T05:09:59.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:58 smithi181 ceph-mon[101970]: failed to encode map e948 with expected crc 2024-03-29T05:09:59.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:58 smithi181 ceph-mon[101970]: failed to encode map e948 with expected crc 2024-03-29T05:09:59.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:58 smithi181 ceph-mon[101970]: failed to encode map e948 with expected crc 2024-03-29T05:09:59.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:58 smithi181 ceph-mon[106466]: pgmap v1151: 105 pgs: 105 active+clean; 807 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 116 KiB/s rd, 33 MiB/s wr, 169 op/s 2024-03-29T05:09:59.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:58 smithi181 ceph-mon[106466]: osdmap e948: 8 total, 8 up, 8 in 2024-03-29T05:09:59.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:58 smithi181 ceph-mon[106466]: failed to encode map e948 with expected crc 2024-03-29T05:09:59.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:58 smithi181 ceph-mon[106466]: failed to encode map e948 with expected crc 2024-03-29T05:09:59.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:58 smithi181 ceph-mon[106466]: failed to encode map e948 with expected crc 2024-03-29T05:09:59.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:58 smithi187 ceph-mon[93536]: pgmap v1151: 105 pgs: 105 active+clean; 807 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 116 KiB/s rd, 33 MiB/s wr, 169 op/s 2024-03-29T05:09:59.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:58 smithi187 ceph-mon[93536]: osdmap e948: 8 total, 8 up, 8 in 2024-03-29T05:09:59.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:58 smithi187 ceph-mon[93536]: failed to encode map e948 with expected crc 2024-03-29T05:09:59.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:58 smithi187 ceph-mon[93536]: failed to encode map e948 with expected crc 2024-03-29T05:09:59.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:58 smithi187 ceph-mon[93536]: failed to encode map e948 with expected crc 2024-03-29T05:09:59.940 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:09:59.940 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:09:59.958 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:09:59.958 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:09:59.958 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:09:59.965 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:09:59.965 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:09:59.965 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:09:59.972 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:09:59.972 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:09:59.972 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:09:59.978 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:09:59.978 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:09:59.978 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:09:59.985 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~109051904,285212672~130023424] 2024-03-29T05:09:59.985 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:09:59.985 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:09:59.991 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:09:59.991 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:09:59.991 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:09:59.996 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:09:59.996 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:09:59.996 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:00.002 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:00.002 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:00.002 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:00.006 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~16777216] 2024-03-29T05:10:00.006 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:00.006 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:00.010 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:10:00.010 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:10:00.010 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:10:00.016 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:10:00.016 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:00.017 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:00.018 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~29360128,100663296~58720256,171966464~20971520,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:00.018 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:00.018 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:00.020 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~8388608,88080384~8388608,100663296~54525952,171966464~20971520,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,360710144~29360128] 2024-03-29T05:10:00.020 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to HEAD diff [4194304~4194304,25165824~16777216,67108864~4194304,92274688~4194304,100663296~37748736,142606336~12582912,180355072~12582912,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:00.020 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,25165824~16777216,67108864~4194304,92274688~4194304,100663296~37748736,142606336~12582912,180355072~12582912,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:00.023 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,25165824~20971520,67108864~8388608,92274688~4194304,100663296~54525952,180355072~12582912,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:00.023 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to HEAD diff [25165824~16777216,67108864~4194304,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,218103808~12582912,234881024~4194304,251658240~20971520,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,348127232~4194304,369098752~12582912,385875968~4194304] 2024-03-29T05:10:00.023 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~16777216,67108864~4194304,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,218103808~12582912,234881024~4194304,251658240~20971520,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,348127232~4194304,369098752~12582912,385875968~4194304] 2024-03-29T05:10:00.024 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~20971520,67108864~8388608,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,197132288~8388608,209715200~33554432,251658240~25165824,293601280~41943040,348127232~4194304,369098752~12582912,385875968~4194304] 2024-03-29T05:10:00.024 INFO:tasks.workunit.client.0.smithi181.stdout:from 15 to HEAD diff [25165824~16777216,92274688~4194304,100663296~8388608,117440512~8388608,130023424~4194304,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,222298112~8388608,234881024~4194304,251658240~12582912,293601280~8388608,306184192~4194304,314572800~4194304,322961408~4194304,331350016~4194304,348127232~4194304,369098752~12582912] 2024-03-29T05:10:00.024 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~16777216,92274688~4194304,100663296~8388608,117440512~8388608,130023424~4194304,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,222298112~8388608,234881024~4194304,251658240~12582912,293601280~8388608,306184192~4194304,314572800~4194304,322961408~4194304,331350016~4194304,348127232~4194304,369098752~12582912] 2024-03-29T05:10:00.026 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~20971520,92274688~4194304,100663296~12582912,117440512~16777216,138412032~16777216,180355072~12582912,197132288~8388608,209715200~8388608,222298112~20971520,251658240~16777216,293601280~41943040,348127232~4194304,369098752~12582912] 2024-03-29T05:10:00.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:59 smithi181 ceph-mon[101970]: failed to encode map e949 with expected crc 2024-03-29T05:10:00.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:59 smithi181 ceph-mon[101970]: osdmap e949: 8 total, 8 up, 8 in 2024-03-29T05:10:00.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:59 smithi181 ceph-mon[101970]: failed to encode map e949 with expected crc 2024-03-29T05:10:00.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:09:59 smithi181 ceph-mon[101970]: failed to encode map e949 with expected crc 2024-03-29T05:10:00.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:59 smithi181 ceph-mon[106466]: failed to encode map e949 with expected crc 2024-03-29T05:10:00.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:59 smithi181 ceph-mon[106466]: osdmap e949: 8 total, 8 up, 8 in 2024-03-29T05:10:00.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:59 smithi181 ceph-mon[106466]: failed to encode map e949 with expected crc 2024-03-29T05:10:00.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:09:59 smithi181 ceph-mon[106466]: failed to encode map e949 with expected crc 2024-03-29T05:10:00.284 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:59 smithi187 ceph-mon[93536]: failed to encode map e949 with expected crc 2024-03-29T05:10:00.284 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:59 smithi187 ceph-mon[93536]: osdmap e949: 8 total, 8 up, 8 in 2024-03-29T05:10:00.285 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:59 smithi187 ceph-mon[93536]: failed to encode map e949 with expected crc 2024-03-29T05:10:00.285 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:09:59 smithi187 ceph-mon[93536]: failed to encode map e949 with expected crc 2024-03-29T05:10:00.938 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:00.939 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:00.962 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:00.962 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:00.962 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:00.968 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:00.968 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:00.968 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:00.974 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:00.974 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:00.974 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:00.980 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:00.980 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:00.980 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:00.985 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~109051904,285212672~130023424] 2024-03-29T05:10:00.985 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:00.985 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:00.991 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:10:00.991 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:00.991 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:00.995 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:00.995 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:00.995 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:01.000 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:01.000 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:01.000 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:01.003 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~16777216] 2024-03-29T05:10:01.004 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:01.004 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:01.006 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:10:01.006 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:10:01.006 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:10:01.009 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:10:01.009 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:01.010 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:01.012 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~29360128,100663296~58720256,171966464~20971520,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:01.012 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:01.012 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:01.014 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~8388608,88080384~8388608,100663296~54525952,171966464~20971520,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,360710144~29360128] 2024-03-29T05:10:01.014 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to HEAD diff [4194304~4194304,25165824~16777216,67108864~4194304,92274688~4194304,100663296~37748736,142606336~12582912,180355072~12582912,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:01.014 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,25165824~16777216,67108864~4194304,92274688~4194304,100663296~37748736,142606336~12582912,180355072~12582912,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:01.016 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,25165824~20971520,67108864~8388608,92274688~4194304,100663296~54525952,180355072~12582912,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:01.016 INFO:tasks.workunit.client.0.smithi181.stdout:from 14 to HEAD diff [25165824~16777216,67108864~4194304,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,218103808~12582912,234881024~4194304,251658240~20971520,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,348127232~4194304,369098752~12582912,385875968~4194304] 2024-03-29T05:10:01.016 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [25165824~16777216,67108864~4194304,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,197132288~4194304,209715200~4194304,218103808~12582912,234881024~4194304,251658240~20971520,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,348127232~4194304,369098752~12582912,385875968~4194304] 2024-03-29T05:10:01.017 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [25165824~20971520,67108864~8388608,92274688~4194304,100663296~33554432,138412032~16777216,180355072~12582912,197132288~8388608,209715200~33554432,251658240~25165824,293601280~41943040,348127232~4194304,369098752~12582912,385875968~4194304] 2024-03-29T05:10:01.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:00 smithi181 ceph-mon[101970]: pgmap v1154: 105 pgs: 105 active+clean; 875 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 441 KiB/s rd, 36 MiB/s wr, 774 op/s 2024-03-29T05:10:01.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:00 smithi181 ceph-mon[101970]: failed to encode map e950 with expected crc 2024-03-29T05:10:01.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:00 smithi181 ceph-mon[101970]: osdmap e950: 8 total, 8 up, 8 in 2024-03-29T05:10:01.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:00 smithi181 ceph-mon[101970]: failed to encode map e950 with expected crc 2024-03-29T05:10:01.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:00 smithi181 ceph-mon[101970]: failed to encode map e950 with expected crc 2024-03-29T05:10:01.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:00 smithi181 ceph-mon[101970]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T05:10:01.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:00 smithi181 ceph-mon[106466]: pgmap v1154: 105 pgs: 105 active+clean; 875 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 441 KiB/s rd, 36 MiB/s wr, 774 op/s 2024-03-29T05:10:01.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:00 smithi181 ceph-mon[106466]: failed to encode map e950 with expected crc 2024-03-29T05:10:01.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:00 smithi181 ceph-mon[106466]: osdmap e950: 8 total, 8 up, 8 in 2024-03-29T05:10:01.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:00 smithi181 ceph-mon[106466]: failed to encode map e950 with expected crc 2024-03-29T05:10:01.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:00 smithi181 ceph-mon[106466]: failed to encode map e950 with expected crc 2024-03-29T05:10:01.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:00 smithi181 ceph-mon[106466]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T05:10:01.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:00 smithi187 ceph-mon[93536]: pgmap v1154: 105 pgs: 105 active+clean; 875 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 441 KiB/s rd, 36 MiB/s wr, 774 op/s 2024-03-29T05:10:01.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:00 smithi187 ceph-mon[93536]: failed to encode map e950 with expected crc 2024-03-29T05:10:01.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:00 smithi187 ceph-mon[93536]: osdmap e950: 8 total, 8 up, 8 in 2024-03-29T05:10:01.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:00 smithi187 ceph-mon[93536]: failed to encode map e950 with expected crc 2024-03-29T05:10:01.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:00 smithi187 ceph-mon[93536]: failed to encode map e950 with expected crc 2024-03-29T05:10:01.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:00 smithi187 ceph-mon[93536]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T05:10:01.945 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:01.945 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:01.967 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:01.967 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:01.967 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:01.973 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:01.973 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:01.973 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:01.980 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:01.980 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:01.980 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:01.986 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:01.986 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:01.986 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:01.991 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~109051904,285212672~130023424] 2024-03-29T05:10:01.992 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:01.992 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:01.996 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:10:01.996 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:01.996 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:02.000 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:02.000 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:02.000 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:02.005 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:02.005 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:02.005 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:02.008 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~16777216] 2024-03-29T05:10:02.008 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:02.008 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:02.010 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:10:02.011 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:10:02.011 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:10:02.014 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:10:02.014 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:02.014 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:02.016 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~29360128,100663296~58720256,171966464~20971520,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:02.016 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:02.016 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:02.018 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~8388608,88080384~8388608,100663296~54525952,171966464~20971520,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,360710144~29360128] 2024-03-29T05:10:02.018 INFO:tasks.workunit.client.0.smithi181.stdout:from 13 to HEAD diff [4194304~4194304,25165824~16777216,67108864~4194304,92274688~4194304,100663296~37748736,142606336~12582912,180355072~12582912,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:02.019 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,25165824~16777216,67108864~4194304,92274688~4194304,100663296~37748736,142606336~12582912,180355072~12582912,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:02.019 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,25165824~20971520,67108864~8388608,92274688~4194304,100663296~54525952,180355072~12582912,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:02.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:01 smithi181 ceph-mon[106466]: failed to encode map e951 with expected crc 2024-03-29T05:10:02.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:01 smithi181 ceph-mon[106466]: osdmap e951: 8 total, 8 up, 8 in 2024-03-29T05:10:02.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:01 smithi181 ceph-mon[106466]: failed to encode map e951 with expected crc 2024-03-29T05:10:02.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:01 smithi181 ceph-mon[106466]: failed to encode map e951 with expected crc 2024-03-29T05:10:02.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:01 smithi181 ceph-mon[106466]: failed to encode map e951 with expected crc 2024-03-29T05:10:02.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:01 smithi181 ceph-mon[101970]: failed to encode map e951 with expected crc 2024-03-29T05:10:02.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:01 smithi181 ceph-mon[101970]: osdmap e951: 8 total, 8 up, 8 in 2024-03-29T05:10:02.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:01 smithi181 ceph-mon[101970]: failed to encode map e951 with expected crc 2024-03-29T05:10:02.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:01 smithi181 ceph-mon[101970]: failed to encode map e951 with expected crc 2024-03-29T05:10:02.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:01 smithi181 ceph-mon[101970]: failed to encode map e951 with expected crc 2024-03-29T05:10:02.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:01 smithi187 ceph-mon[93536]: failed to encode map e951 with expected crc 2024-03-29T05:10:02.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:01 smithi187 ceph-mon[93536]: osdmap e951: 8 total, 8 up, 8 in 2024-03-29T05:10:02.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:01 smithi187 ceph-mon[93536]: failed to encode map e951 with expected crc 2024-03-29T05:10:02.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:01 smithi187 ceph-mon[93536]: failed to encode map e951 with expected crc 2024-03-29T05:10:02.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:01 smithi187 ceph-mon[93536]: failed to encode map e951 with expected crc 2024-03-29T05:10:02.954 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:02.954 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:02.976 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:02.976 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:02.976 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:02.981 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:02.981 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:02.981 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:02.986 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:02.986 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:02.986 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:02.990 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:02.990 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:02.990 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:02.994 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~109051904,285212672~130023424] 2024-03-29T05:10:02.994 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:02.994 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:02.998 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:10:02.998 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:02.998 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:03.002 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:03.002 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:03.002 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:03.005 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:03.005 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:03.005 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:03.008 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~16777216] 2024-03-29T05:10:03.008 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:03.008 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:03.010 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:10:03.011 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:10:03.011 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:10:03.012 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:10:03.012 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:03.013 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:03.014 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~29360128,100663296~58720256,171966464~20971520,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:03.014 INFO:tasks.workunit.client.0.smithi181.stdout:from 12 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:03.014 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~4194304,88080384~8388608,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~12582912,234881024~4194304,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,360710144~20971520,385875968~4194304] 2024-03-29T05:10:03.015 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~8388608,88080384~8388608,100663296~54525952,171966464~20971520,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,360710144~29360128] 2024-03-29T05:10:03.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:02 smithi181 ceph-mon[106466]: pgmap v1157: 105 pgs: 105 active+clean; 875 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 441 KiB/s rd, 17 MiB/s wr, 764 op/s 2024-03-29T05:10:03.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:02 smithi181 ceph-mon[106466]: failed to encode map e952 with expected crc 2024-03-29T05:10:03.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:02 smithi181 ceph-mon[106466]: failed to encode map e952 with expected crc 2024-03-29T05:10:03.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:02 smithi181 ceph-mon[106466]: failed to encode map e952 with expected crc 2024-03-29T05:10:03.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:02 smithi181 ceph-mon[106466]: osdmap e952: 8 total, 8 up, 8 in 2024-03-29T05:10:03.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:02 smithi181 ceph-mon[106466]: failed to encode map e952 with expected crc 2024-03-29T05:10:03.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:02 smithi181 ceph-mon[106466]: failed to encode map e952 with expected crc 2024-03-29T05:10:03.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:02 smithi181 ceph-mon[101970]: pgmap v1157: 105 pgs: 105 active+clean; 875 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 441 KiB/s rd, 17 MiB/s wr, 764 op/s 2024-03-29T05:10:03.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:02 smithi181 ceph-mon[101970]: failed to encode map e952 with expected crc 2024-03-29T05:10:03.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:02 smithi181 ceph-mon[101970]: failed to encode map e952 with expected crc 2024-03-29T05:10:03.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:02 smithi181 ceph-mon[101970]: failed to encode map e952 with expected crc 2024-03-29T05:10:03.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:02 smithi181 ceph-mon[101970]: osdmap e952: 8 total, 8 up, 8 in 2024-03-29T05:10:03.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:02 smithi181 ceph-mon[101970]: failed to encode map e952 with expected crc 2024-03-29T05:10:03.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:02 smithi181 ceph-mon[101970]: failed to encode map e952 with expected crc 2024-03-29T05:10:03.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:02 smithi187 ceph-mon[93536]: pgmap v1157: 105 pgs: 105 active+clean; 875 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 441 KiB/s rd, 17 MiB/s wr, 764 op/s 2024-03-29T05:10:03.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:02 smithi187 ceph-mon[93536]: failed to encode map e952 with expected crc 2024-03-29T05:10:03.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:02 smithi187 ceph-mon[93536]: failed to encode map e952 with expected crc 2024-03-29T05:10:03.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:02 smithi187 ceph-mon[93536]: failed to encode map e952 with expected crc 2024-03-29T05:10:03.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:02 smithi187 ceph-mon[93536]: osdmap e952: 8 total, 8 up, 8 in 2024-03-29T05:10:03.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:02 smithi187 ceph-mon[93536]: failed to encode map e952 with expected crc 2024-03-29T05:10:03.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:02 smithi187 ceph-mon[93536]: failed to encode map e952 with expected crc 2024-03-29T05:10:03.963 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:03.963 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:03.984 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:03.984 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:03.984 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:03.989 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:03.989 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:03.989 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:03.994 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:03.995 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:03.995 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:03.999 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:03.999 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:03.999 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:04.003 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~109051904,285212672~130023424] 2024-03-29T05:10:04.003 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:04.003 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:04.006 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:10:04.006 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:04.006 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:04.010 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:04.010 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:04.010 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:04.013 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:04.013 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:04.013 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:04.015 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~16777216] 2024-03-29T05:10:04.015 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:04.015 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:04.017 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:10:04.017 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:10:04.017 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:10:04.019 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:10:04.019 INFO:tasks.workunit.client.0.smithi181.stdout:from 11 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:04.019 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~8388608,79691776~16777216,100663296~37748736,142606336~12582912,171966464~20971520,197132288~4194304,205520896~8388608,218103808~20971520,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~4194304,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:04.020 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~37748736,67108864~29360128,100663296~58720256,171966464~20971520,197132288~46137344,247463936~29360128,293601280~46137344,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:04.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:03 smithi181 ceph-mon[101970]: failed to encode map e953 with expected crc 2024-03-29T05:10:04.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:03 smithi181 ceph-mon[101970]: osdmap e953: 8 total, 8 up, 8 in 2024-03-29T05:10:04.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:03 smithi181 ceph-mon[101970]: failed to encode map e953 with expected crc 2024-03-29T05:10:04.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:03 smithi181 ceph-mon[101970]: failed to encode map e953 with expected crc 2024-03-29T05:10:04.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:03 smithi181 ceph-mon[101970]: failed to encode map e953 with expected crc 2024-03-29T05:10:04.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:03 smithi181 ceph-mon[106466]: failed to encode map e953 with expected crc 2024-03-29T05:10:04.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:03 smithi181 ceph-mon[106466]: osdmap e953: 8 total, 8 up, 8 in 2024-03-29T05:10:04.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:03 smithi181 ceph-mon[106466]: failed to encode map e953 with expected crc 2024-03-29T05:10:04.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:03 smithi181 ceph-mon[106466]: failed to encode map e953 with expected crc 2024-03-29T05:10:04.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:03 smithi181 ceph-mon[106466]: failed to encode map e953 with expected crc 2024-03-29T05:10:04.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:03 smithi187 ceph-mon[93536]: failed to encode map e953 with expected crc 2024-03-29T05:10:04.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:03 smithi187 ceph-mon[93536]: osdmap e953: 8 total, 8 up, 8 in 2024-03-29T05:10:04.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:03 smithi187 ceph-mon[93536]: failed to encode map e953 with expected crc 2024-03-29T05:10:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:03 smithi187 ceph-mon[93536]: failed to encode map e953 with expected crc 2024-03-29T05:10:04.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:03 smithi187 ceph-mon[93536]: failed to encode map e953 with expected crc 2024-03-29T05:10:04.971 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:04.972 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:04.990 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:04.991 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:04.991 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:04.995 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:04.995 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:04.995 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:04.999 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:04.999 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:04.999 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:05.003 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:05.003 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:05.003 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:05.007 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~109051904,285212672~130023424] 2024-03-29T05:10:05.007 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:05.007 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:05.010 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:10:05.010 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:05.010 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:05.013 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:05.013 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:05.013 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:05.016 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:05.016 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:05.016 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:05.018 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~16777216] 2024-03-29T05:10:05.018 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:05.018 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:05.019 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:10:05.019 INFO:tasks.workunit.client.0.smithi181.stdout:from 10 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:10:05.019 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~8388608,306184192~4194304,314572800~12582912,331350016~20971520,356515840~33554432,398458880~8388608] 2024-03-29T05:10:05.020 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:10:05.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:04 smithi187 ceph-mon[93536]: pgmap v1160: 105 pgs: 105 active+clean; 862 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 381 KiB/s rd, 1.5 KiB/s wr, 763 op/s 2024-03-29T05:10:05.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:04 smithi187 ceph-mon[93536]: failed to encode map e954 with expected crc 2024-03-29T05:10:05.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:04 smithi187 ceph-mon[93536]: failed to encode map e954 with expected crc 2024-03-29T05:10:05.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:04 smithi187 ceph-mon[93536]: failed to encode map e954 with expected crc 2024-03-29T05:10:05.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:04 smithi187 ceph-mon[93536]: osdmap e954: 8 total, 8 up, 8 in 2024-03-29T05:10:05.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:04 smithi187 ceph-mon[93536]: failed to encode map e954 with expected crc 2024-03-29T05:10:05.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:04 smithi181 ceph-mon[106466]: pgmap v1160: 105 pgs: 105 active+clean; 862 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 381 KiB/s rd, 1.5 KiB/s wr, 763 op/s 2024-03-29T05:10:05.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:04 smithi181 ceph-mon[106466]: failed to encode map e954 with expected crc 2024-03-29T05:10:05.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:04 smithi181 ceph-mon[106466]: failed to encode map e954 with expected crc 2024-03-29T05:10:05.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:04 smithi181 ceph-mon[106466]: failed to encode map e954 with expected crc 2024-03-29T05:10:05.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:04 smithi181 ceph-mon[106466]: osdmap e954: 8 total, 8 up, 8 in 2024-03-29T05:10:05.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:04 smithi181 ceph-mon[106466]: failed to encode map e954 with expected crc 2024-03-29T05:10:05.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:04 smithi181 ceph-mon[101970]: pgmap v1160: 105 pgs: 105 active+clean; 862 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 381 KiB/s rd, 1.5 KiB/s wr, 763 op/s 2024-03-29T05:10:05.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:04 smithi181 ceph-mon[101970]: failed to encode map e954 with expected crc 2024-03-29T05:10:05.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:04 smithi181 ceph-mon[101970]: failed to encode map e954 with expected crc 2024-03-29T05:10:05.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:04 smithi181 ceph-mon[101970]: failed to encode map e954 with expected crc 2024-03-29T05:10:05.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:04 smithi181 ceph-mon[101970]: osdmap e954: 8 total, 8 up, 8 in 2024-03-29T05:10:05.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:04 smithi181 ceph-mon[101970]: failed to encode map e954 with expected crc 2024-03-29T05:10:05.985 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:05.986 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:06.005 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:06.005 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:06.005 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:06.009 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:06.009 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:06.009 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:06.013 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:06.014 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:06.014 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:06.017 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:06.018 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:06.018 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:06.021 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~109051904,285212672~130023424] 2024-03-29T05:10:06.021 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:06.022 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:06.024 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:10:06.024 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:06.024 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:06.026 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:06.027 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:06.027 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:06.029 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:06.029 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:06.029 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:06.031 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~16777216] 2024-03-29T05:10:06.031 INFO:tasks.workunit.client.0.smithi181.stdout:from 9 to HEAD diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:06.031 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~4194304,16777216~25165824,46137344~8388608,67108864~16777216,88080384~8388608,100663296~37748736,142606336~20971520,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~8388608] 2024-03-29T05:10:06.032 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~4194304,16777216~41943040,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~12582912] 2024-03-29T05:10:06.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:05 smithi187 ceph-mon[93536]: failed to encode map e955 with expected crc 2024-03-29T05:10:06.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:05 smithi187 ceph-mon[93536]: failed to encode map e955 with expected crc 2024-03-29T05:10:06.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:05 smithi187 ceph-mon[93536]: osdmap e955: 8 total, 8 up, 8 in 2024-03-29T05:10:06.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:05 smithi181 ceph-mon[106466]: failed to encode map e955 with expected crc 2024-03-29T05:10:06.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:05 smithi181 ceph-mon[106466]: failed to encode map e955 with expected crc 2024-03-29T05:10:06.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:05 smithi181 ceph-mon[106466]: osdmap e955: 8 total, 8 up, 8 in 2024-03-29T05:10:06.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:05 smithi181 ceph-mon[101970]: failed to encode map e955 with expected crc 2024-03-29T05:10:06.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:05 smithi181 ceph-mon[101970]: failed to encode map e955 with expected crc 2024-03-29T05:10:06.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:05 smithi181 ceph-mon[101970]: osdmap e955: 8 total, 8 up, 8 in 2024-03-29T05:10:06.996 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:06.996 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:07.016 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:07.016 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:07.016 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:07.020 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:07.020 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:07.020 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:07.024 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:07.024 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:07.024 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:07.027 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:07.027 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:07.027 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:07.031 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~109051904,285212672~130023424] 2024-03-29T05:10:07.031 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:07.031 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:07.033 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:10:07.033 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:07.033 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:07.035 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:07.035 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:07.035 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:07.037 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:07.037 INFO:tasks.workunit.client.0.smithi181.stdout:from 8 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:07.037 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~8388608,356515840~33554432,398458880~12582912] 2024-03-29T05:10:07.038 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~58720256,356515840~33554432,398458880~16777216] 2024-03-29T05:10:07.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:06 smithi187 ceph-mon[93536]: pgmap v1163: 105 pgs: 105 active+clean; 847 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 563 KiB/s rd, 6.5 KiB/s wr, 1.04k op/s 2024-03-29T05:10:07.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:06 smithi187 ceph-mon[93536]: failed to encode map e956 with expected crc 2024-03-29T05:10:07.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:06 smithi187 ceph-mon[93536]: osdmap e956: 8 total, 8 up, 8 in 2024-03-29T05:10:07.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:06 smithi187 ceph-mon[93536]: failed to encode map e956 with expected crc 2024-03-29T05:10:07.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:06 smithi181 ceph-mon[106466]: pgmap v1163: 105 pgs: 105 active+clean; 847 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 563 KiB/s rd, 6.5 KiB/s wr, 1.04k op/s 2024-03-29T05:10:07.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:06 smithi181 ceph-mon[106466]: failed to encode map e956 with expected crc 2024-03-29T05:10:07.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:06 smithi181 ceph-mon[106466]: osdmap e956: 8 total, 8 up, 8 in 2024-03-29T05:10:07.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:06 smithi181 ceph-mon[106466]: failed to encode map e956 with expected crc 2024-03-29T05:10:07.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:06 smithi181 ceph-mon[101970]: pgmap v1163: 105 pgs: 105 active+clean; 847 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 563 KiB/s rd, 6.5 KiB/s wr, 1.04k op/s 2024-03-29T05:10:07.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:06 smithi181 ceph-mon[101970]: failed to encode map e956 with expected crc 2024-03-29T05:10:07.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:06 smithi181 ceph-mon[101970]: osdmap e956: 8 total, 8 up, 8 in 2024-03-29T05:10:07.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:06 smithi181 ceph-mon[101970]: failed to encode map e956 with expected crc 2024-03-29T05:10:07.745 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:07.745 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:07.767 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:07.767 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:07.767 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:07.769 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:07.770 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:07.770 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:07.774 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:07.774 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:07.774 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:07.777 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:07.777 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:07.777 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:07.779 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~109051904,285212672~130023424] 2024-03-29T05:10:07.779 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:07.779 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:07.782 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:10:07.782 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:07.782 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:07.786 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:07.786 INFO:tasks.workunit.client.0.smithi181.stdout:from 7 to HEAD diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:07.786 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~20971520,92274688~4194304,100663296~62914560,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~16777216,314572800~25165824,343932928~46137344,398458880~12582912] 2024-03-29T05:10:07.786 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [4194304~58720256,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:08.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:07 smithi187 ceph-mon[93536]: failed to encode map e957 with expected crc 2024-03-29T05:10:08.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:07 smithi187 ceph-mon[93536]: osdmap e957: 8 total, 8 up, 8 in 2024-03-29T05:10:08.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:07 smithi187 ceph-mon[93536]: failed to encode map e957 with expected crc 2024-03-29T05:10:08.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:07 smithi187 ceph-mon[93536]: failed to encode map e957 with expected crc 2024-03-29T05:10:08.384 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:07 smithi187 ceph-mon[93536]: osdmap e958: 8 total, 8 up, 8 in 2024-03-29T05:10:08.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:07 smithi181 ceph-mon[101970]: failed to encode map e957 with expected crc 2024-03-29T05:10:08.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:07 smithi181 ceph-mon[101970]: osdmap e957: 8 total, 8 up, 8 in 2024-03-29T05:10:08.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:07 smithi181 ceph-mon[101970]: failed to encode map e957 with expected crc 2024-03-29T05:10:08.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:07 smithi181 ceph-mon[101970]: failed to encode map e957 with expected crc 2024-03-29T05:10:08.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:07 smithi181 ceph-mon[101970]: osdmap e958: 8 total, 8 up, 8 in 2024-03-29T05:10:08.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:07 smithi181 ceph-mon[106466]: failed to encode map e957 with expected crc 2024-03-29T05:10:08.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:07 smithi181 ceph-mon[106466]: osdmap e957: 8 total, 8 up, 8 in 2024-03-29T05:10:08.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:07 smithi181 ceph-mon[106466]: failed to encode map e957 with expected crc 2024-03-29T05:10:08.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:07 smithi181 ceph-mon[106466]: failed to encode map e957 with expected crc 2024-03-29T05:10:08.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:07 smithi181 ceph-mon[106466]: osdmap e958: 8 total, 8 up, 8 in 2024-03-29T05:10:08.746 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:08.746 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:08.764 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:08.764 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:08.764 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:08.768 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:08.768 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:08.768 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:08.771 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:08.771 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:08.771 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:08.773 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:08.773 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:08.773 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:08.775 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~109051904,285212672~130023424] 2024-03-29T05:10:08.775 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:08.775 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:08.777 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:10:08.777 INFO:tasks.workunit.client.0.smithi181.stdout:from 6 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:08.777 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,293601280~46137344,343932928~46137344,398458880~12582912] 2024-03-29T05:10:08.778 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~29360128,293601280~96468992,398458880~16777216] 2024-03-29T05:10:09.203 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:10:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:10:08] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.43.0" 2024-03-29T05:10:09.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:08 smithi181 ceph-mon[106466]: pgmap v1166: 105 pgs: 105 active+clean; 800 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 228 KiB/s rd, 5.0 KiB/s wr, 371 op/s 2024-03-29T05:10:09.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:08 smithi181 ceph-mon[106466]: failed to encode map e958 with expected crc 2024-03-29T05:10:09.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:08 smithi181 ceph-mon[106466]: failed to encode map e958 with expected crc 2024-03-29T05:10:09.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:08 smithi181 ceph-mon[106466]: failed to encode map e958 with expected crc 2024-03-29T05:10:09.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:08 smithi181 ceph-mon[106466]: failed to encode map e958 with expected crc 2024-03-29T05:10:09.204 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:08 smithi181 ceph-mon[106466]: osdmap e959: 8 total, 8 up, 8 in 2024-03-29T05:10:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:08 smithi181 ceph-mon[101970]: pgmap v1166: 105 pgs: 105 active+clean; 800 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 228 KiB/s rd, 5.0 KiB/s wr, 371 op/s 2024-03-29T05:10:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:08 smithi181 ceph-mon[101970]: failed to encode map e958 with expected crc 2024-03-29T05:10:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:08 smithi181 ceph-mon[101970]: failed to encode map e958 with expected crc 2024-03-29T05:10:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:08 smithi181 ceph-mon[101970]: failed to encode map e958 with expected crc 2024-03-29T05:10:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:08 smithi181 ceph-mon[101970]: failed to encode map e958 with expected crc 2024-03-29T05:10:09.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:08 smithi181 ceph-mon[101970]: osdmap e959: 8 total, 8 up, 8 in 2024-03-29T05:10:09.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:08 smithi187 ceph-mon[93536]: pgmap v1166: 105 pgs: 105 active+clean; 800 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 228 KiB/s rd, 5.0 KiB/s wr, 371 op/s 2024-03-29T05:10:09.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:08 smithi187 ceph-mon[93536]: failed to encode map e958 with expected crc 2024-03-29T05:10:09.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:08 smithi187 ceph-mon[93536]: failed to encode map e958 with expected crc 2024-03-29T05:10:09.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:08 smithi187 ceph-mon[93536]: failed to encode map e958 with expected crc 2024-03-29T05:10:09.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:08 smithi187 ceph-mon[93536]: failed to encode map e958 with expected crc 2024-03-29T05:10:09.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:08 smithi187 ceph-mon[93536]: osdmap e959: 8 total, 8 up, 8 in 2024-03-29T05:10:09.748 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:09.748 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:09.766 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:09.767 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:09.767 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:09.770 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:09.770 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:09.770 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:09.772 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:09.772 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:09.773 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:09.774 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:09.775 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:09.775 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:09.776 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~109051904,285212672~130023424] 2024-03-29T05:10:09.776 INFO:tasks.workunit.client.0.smithi181.stdout:from 5 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:09.776 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,247463936~25165824,276824064~4194304,285212672~54525952,343932928~46137344,398458880~12582912] 2024-03-29T05:10:09.777 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~71303168,247463936~33554432,285212672~104857600,398458880~16777216] 2024-03-29T05:10:10.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:09 smithi187 ceph-mon[93536]: failed to encode map e959 with expected crc 2024-03-29T05:10:10.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:09 smithi187 ceph-mon[93536]: failed to encode map e959 with expected crc 2024-03-29T05:10:10.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:09 smithi187 ceph-mon[93536]: failed to encode map e959 with expected crc 2024-03-29T05:10:10.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:09 smithi187 ceph-mon[93536]: osdmap e960: 8 total, 8 up, 8 in 2024-03-29T05:10:10.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:09 smithi181 ceph-mon[106466]: failed to encode map e959 with expected crc 2024-03-29T05:10:10.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:09 smithi181 ceph-mon[106466]: failed to encode map e959 with expected crc 2024-03-29T05:10:10.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:09 smithi181 ceph-mon[106466]: failed to encode map e959 with expected crc 2024-03-29T05:10:10.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:09 smithi181 ceph-mon[106466]: osdmap e960: 8 total, 8 up, 8 in 2024-03-29T05:10:10.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:09 smithi181 ceph-mon[101970]: failed to encode map e959 with expected crc 2024-03-29T05:10:10.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:09 smithi181 ceph-mon[101970]: failed to encode map e959 with expected crc 2024-03-29T05:10:10.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:09 smithi181 ceph-mon[101970]: failed to encode map e959 with expected crc 2024-03-29T05:10:10.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:09 smithi181 ceph-mon[101970]: osdmap e960: 8 total, 8 up, 8 in 2024-03-29T05:10:10.749 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:10.749 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:10.767 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:10.767 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:10.767 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:10.770 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:10.771 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:10.771 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:10.773 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:10.773 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:10.773 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:10.775 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:10.775 INFO:tasks.workunit.client.0.smithi181.stdout:from 4 to HEAD diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:10.775 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~16777216,67108864~20971520,92274688~4194304,100663296~58720256,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~54525952,343932928~50331648,398458880~12582912] 2024-03-29T05:10:10.777 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~62914560,67108864~29360128,100663296~62914560,171966464~109051904,285212672~130023424] 2024-03-29T05:10:11.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:10 smithi187 ceph-mon[93536]: pgmap v1169: 105 pgs: 105 active+clean; 752 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 213 KiB/s rd, 6.0 KiB/s wr, 362 op/s 2024-03-29T05:10:11.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:10 smithi187 ceph-mon[93536]: failed to encode map e960 with expected crc 2024-03-29T05:10:11.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:10 smithi187 ceph-mon[93536]: failed to encode map e960 with expected crc 2024-03-29T05:10:11.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:10 smithi187 ceph-mon[93536]: osdmap e961: 8 total, 8 up, 8 in 2024-03-29T05:10:11.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:10 smithi181 ceph-mon[101970]: pgmap v1169: 105 pgs: 105 active+clean; 752 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 213 KiB/s rd, 6.0 KiB/s wr, 362 op/s 2024-03-29T05:10:11.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:10 smithi181 ceph-mon[101970]: failed to encode map e960 with expected crc 2024-03-29T05:10:11.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:10 smithi181 ceph-mon[101970]: failed to encode map e960 with expected crc 2024-03-29T05:10:11.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:10 smithi181 ceph-mon[101970]: osdmap e961: 8 total, 8 up, 8 in 2024-03-29T05:10:11.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:10 smithi181 ceph-mon[106466]: pgmap v1169: 105 pgs: 105 active+clean; 752 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 213 KiB/s rd, 6.0 KiB/s wr, 362 op/s 2024-03-29T05:10:11.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:10 smithi181 ceph-mon[106466]: failed to encode map e960 with expected crc 2024-03-29T05:10:11.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:10 smithi181 ceph-mon[106466]: failed to encode map e960 with expected crc 2024-03-29T05:10:11.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:10 smithi181 ceph-mon[106466]: osdmap e961: 8 total, 8 up, 8 in 2024-03-29T05:10:11.748 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:11.748 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:11.769 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:11.769 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:11.769 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:11.772 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:11.772 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:11.772 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:11.773 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:11.773 INFO:tasks.workunit.client.0.smithi181.stdout:from 3 to HEAD diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:11.773 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~41943040,46137344~41943040,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:11.774 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:11 smithi187 ceph-mon[93536]: failed to encode map e961 with expected crc 2024-03-29T05:10:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:11 smithi187 ceph-mon[93536]: failed to encode map e961 with expected crc 2024-03-29T05:10:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:11 smithi187 ceph-mon[93536]: failed to encode map e961 with expected crc 2024-03-29T05:10:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:11 smithi187 ceph-mon[93536]: failed to encode map e961 with expected crc 2024-03-29T05:10:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:10:12.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:11 smithi187 ceph-mon[93536]: osdmap e962: 8 total, 8 up, 8 in 2024-03-29T05:10:12.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:11 smithi181 ceph-mon[106466]: failed to encode map e961 with expected crc 2024-03-29T05:10:12.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:11 smithi181 ceph-mon[106466]: failed to encode map e961 with expected crc 2024-03-29T05:10:12.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:11 smithi181 ceph-mon[106466]: failed to encode map e961 with expected crc 2024-03-29T05:10:12.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:11 smithi181 ceph-mon[106466]: failed to encode map e961 with expected crc 2024-03-29T05:10:12.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:10:12.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:11 smithi181 ceph-mon[106466]: osdmap e962: 8 total, 8 up, 8 in 2024-03-29T05:10:12.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:11 smithi181 ceph-mon[101970]: failed to encode map e961 with expected crc 2024-03-29T05:10:12.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:11 smithi181 ceph-mon[101970]: failed to encode map e961 with expected crc 2024-03-29T05:10:12.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:11 smithi181 ceph-mon[101970]: failed to encode map e961 with expected crc 2024-03-29T05:10:12.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:11 smithi181 ceph-mon[101970]: failed to encode map e961 with expected crc 2024-03-29T05:10:12.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:10:12.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:11 smithi181 ceph-mon[101970]: osdmap e962: 8 total, 8 up, 8 in 2024-03-29T05:10:12.747 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:12.747 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:12.762 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:12.762 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:12.762 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:12.764 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:12.764 INFO:tasks.workunit.client.0.smithi181.stdout:from 2 to HEAD diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:12.764 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~67108864,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~4194304,285212672~25165824,314572800~25165824,343932928~50331648,398458880~12582912] 2024-03-29T05:10:12.766 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~163577856,171966464~109051904,285212672~130023424] 2024-03-29T05:10:13.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:12 smithi187 ceph-mon[93536]: pgmap v1172: 105 pgs: 105 active+clean; 752 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 167 KiB/s rd, 6.0 KiB/s wr, 270 op/s 2024-03-29T05:10:13.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:12 smithi187 ceph-mon[93536]: failed to encode map e962 with expected crc 2024-03-29T05:10:13.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:12 smithi187 ceph-mon[93536]: failed to encode map e962 with expected crc 2024-03-29T05:10:13.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:12 smithi187 ceph-mon[93536]: failed to encode map e962 with expected crc 2024-03-29T05:10:13.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:12 smithi187 ceph-mon[93536]: failed to encode map e962 with expected crc 2024-03-29T05:10:13.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:12 smithi187 ceph-mon[93536]: failed to encode map e962 with expected crc 2024-03-29T05:10:13.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:12 smithi187 ceph-mon[93536]: osdmap e963: 8 total, 8 up, 8 in 2024-03-29T05:10:13.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:12 smithi181 ceph-mon[101970]: pgmap v1172: 105 pgs: 105 active+clean; 752 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 167 KiB/s rd, 6.0 KiB/s wr, 270 op/s 2024-03-29T05:10:13.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:12 smithi181 ceph-mon[101970]: failed to encode map e962 with expected crc 2024-03-29T05:10:13.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:12 smithi181 ceph-mon[101970]: failed to encode map e962 with expected crc 2024-03-29T05:10:13.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:12 smithi181 ceph-mon[101970]: failed to encode map e962 with expected crc 2024-03-29T05:10:13.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:12 smithi181 ceph-mon[101970]: failed to encode map e962 with expected crc 2024-03-29T05:10:13.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:12 smithi181 ceph-mon[101970]: failed to encode map e962 with expected crc 2024-03-29T05:10:13.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:12 smithi181 ceph-mon[101970]: osdmap e963: 8 total, 8 up, 8 in 2024-03-29T05:10:13.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:12 smithi181 ceph-mon[106466]: pgmap v1172: 105 pgs: 105 active+clean; 752 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 167 KiB/s rd, 6.0 KiB/s wr, 270 op/s 2024-03-29T05:10:13.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:12 smithi181 ceph-mon[106466]: failed to encode map e962 with expected crc 2024-03-29T05:10:13.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:12 smithi181 ceph-mon[106466]: failed to encode map e962 with expected crc 2024-03-29T05:10:13.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:12 smithi181 ceph-mon[106466]: failed to encode map e962 with expected crc 2024-03-29T05:10:13.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:12 smithi181 ceph-mon[106466]: failed to encode map e962 with expected crc 2024-03-29T05:10:13.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:12 smithi181 ceph-mon[106466]: failed to encode map e962 with expected crc 2024-03-29T05:10:13.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:12 smithi181 ceph-mon[106466]: osdmap e963: 8 total, 8 up, 8 in 2024-03-29T05:10:13.745 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:13.745 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:13.758 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:13.758 INFO:tasks.workunit.client.0.smithi181.stdout:from 1 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:13.758 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:13.760 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:14.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:13 smithi187 ceph-mon[93536]: failed to encode map e963 with expected crc 2024-03-29T05:10:14.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:13 smithi187 ceph-mon[93536]: failed to encode map e963 with expected crc 2024-03-29T05:10:14.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:13 smithi187 ceph-mon[93536]: osdmap e964: 8 total, 8 up, 8 in 2024-03-29T05:10:14.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:13 smithi181 ceph-mon[101970]: failed to encode map e963 with expected crc 2024-03-29T05:10:14.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:13 smithi181 ceph-mon[101970]: failed to encode map e963 with expected crc 2024-03-29T05:10:14.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:13 smithi181 ceph-mon[101970]: osdmap e964: 8 total, 8 up, 8 in 2024-03-29T05:10:14.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:13 smithi181 ceph-mon[106466]: failed to encode map e963 with expected crc 2024-03-29T05:10:14.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:13 smithi181 ceph-mon[106466]: failed to encode map e963 with expected crc 2024-03-29T05:10:14.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:13 smithi181 ceph-mon[106466]: osdmap e964: 8 total, 8 up, 8 in 2024-03-29T05:10:14.749 INFO:tasks.workunit.client.0.smithi181.stdout:from 0 to HEAD diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:14.750 INFO:tasks.workunit.client.0.smithi181.stdout: limited diff [0~88080384,92274688~71303168,171966464~20971520,197132288~4194304,205520896~33554432,243269632~29360128,276824064~117440512,398458880~20971520] 2024-03-29T05:10:14.763 INFO:tasks.workunit.client.0.smithi181.stdout: actual was [0~167772160,171966464~247463936] 2024-03-29T05:10:15.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:15 smithi187 ceph-mon[93536]: pgmap v1175: 105 pgs: 105 active+clean; 636 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 42 KiB/s rd, 1.2 KiB/s wr, 84 op/s 2024-03-29T05:10:15.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:15 smithi187 ceph-mon[93536]: failed to encode map e964 with expected crc 2024-03-29T05:10:15.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:15 smithi187 ceph-mon[93536]: failed to encode map e964 with expected crc 2024-03-29T05:10:15.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:15 smithi187 ceph-mon[93536]: failed to encode map e964 with expected crc 2024-03-29T05:10:15.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:15 smithi187 ceph-mon[93536]: failed to encode map e964 with expected crc 2024-03-29T05:10:15.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:15 smithi187 ceph-mon[93536]: osdmap e965: 8 total, 8 up, 8 in 2024-03-29T05:10:15.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:15 smithi181 ceph-mon[101970]: pgmap v1175: 105 pgs: 105 active+clean; 636 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 42 KiB/s rd, 1.2 KiB/s wr, 84 op/s 2024-03-29T05:10:15.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:15 smithi181 ceph-mon[101970]: failed to encode map e964 with expected crc 2024-03-29T05:10:15.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:15 smithi181 ceph-mon[101970]: failed to encode map e964 with expected crc 2024-03-29T05:10:15.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:15 smithi181 ceph-mon[101970]: failed to encode map e964 with expected crc 2024-03-29T05:10:15.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:15 smithi181 ceph-mon[101970]: failed to encode map e964 with expected crc 2024-03-29T05:10:15.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:15 smithi181 ceph-mon[101970]: osdmap e965: 8 total, 8 up, 8 in 2024-03-29T05:10:15.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:15 smithi181 ceph-mon[106466]: pgmap v1175: 105 pgs: 105 active+clean; 636 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 42 KiB/s rd, 1.2 KiB/s wr, 84 op/s 2024-03-29T05:10:15.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:15 smithi181 ceph-mon[106466]: failed to encode map e964 with expected crc 2024-03-29T05:10:15.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:15 smithi181 ceph-mon[106466]: failed to encode map e964 with expected crc 2024-03-29T05:10:15.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:15 smithi181 ceph-mon[106466]: failed to encode map e964 with expected crc 2024-03-29T05:10:15.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:15 smithi181 ceph-mon[106466]: failed to encode map e964 with expected crc 2024-03-29T05:10:15.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:15 smithi181 ceph-mon[106466]: osdmap e965: 8 total, 8 up, 8 in 2024-03-29T05:10:15.765 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/1.DiffIterateStress (38945 ms) 2024-03-29T05:10:15.765 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParent 2024-03-29T05:10:15.765 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:10:16.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:16 smithi187 ceph-mon[93536]: failed to encode map e965 with expected crc 2024-03-29T05:10:16.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:16 smithi187 ceph-mon[93536]: failed to encode map e965 with expected crc 2024-03-29T05:10:16.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:16 smithi187 ceph-mon[93536]: failed to encode map e965 with expected crc 2024-03-29T05:10:16.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:16 smithi187 ceph-mon[93536]: osdmap e966: 8 total, 8 up, 8 in 2024-03-29T05:10:16.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:16 smithi181 ceph-mon[101970]: failed to encode map e965 with expected crc 2024-03-29T05:10:16.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:16 smithi181 ceph-mon[101970]: failed to encode map e965 with expected crc 2024-03-29T05:10:16.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:16 smithi181 ceph-mon[101970]: failed to encode map e965 with expected crc 2024-03-29T05:10:16.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:16 smithi181 ceph-mon[101970]: osdmap e966: 8 total, 8 up, 8 in 2024-03-29T05:10:16.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:16 smithi181 ceph-mon[106466]: failed to encode map e965 with expected crc 2024-03-29T05:10:16.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:16 smithi181 ceph-mon[106466]: failed to encode map e965 with expected crc 2024-03-29T05:10:16.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:16 smithi181 ceph-mon[106466]: failed to encode map e965 with expected crc 2024-03-29T05:10:16.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:16 smithi181 ceph-mon[106466]: osdmap e966: 8 total, 8 up, 8 in 2024-03-29T05:10:16.856 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/1.DiffIterateParent (1091 ms) 2024-03-29T05:10:16.856 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/1.DiffIterateIgnoreParent 2024-03-29T05:10:16.856 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:10:17.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:17 smithi187 ceph-mon[93536]: pgmap v1178: 105 pgs: 105 active+clean; 600 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 6.2 KiB/s wr, 196 op/s 2024-03-29T05:10:17.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:17 smithi187 ceph-mon[93536]: failed to encode map e966 with expected crc 2024-03-29T05:10:17.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:17 smithi187 ceph-mon[93536]: failed to encode map e966 with expected crc 2024-03-29T05:10:17.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:17 smithi187 ceph-mon[93536]: failed to encode map e966 with expected crc 2024-03-29T05:10:17.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:17 smithi187 ceph-mon[93536]: failed to encode map e966 with expected crc 2024-03-29T05:10:17.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:17 smithi187 ceph-mon[93536]: failed to encode map e966 with expected crc 2024-03-29T05:10:17.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:17 smithi187 ceph-mon[93536]: osdmap e967: 8 total, 8 up, 8 in 2024-03-29T05:10:17.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:17 smithi181 ceph-mon[106466]: pgmap v1178: 105 pgs: 105 active+clean; 600 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 6.2 KiB/s wr, 196 op/s 2024-03-29T05:10:17.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:17 smithi181 ceph-mon[106466]: failed to encode map e966 with expected crc 2024-03-29T05:10:17.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:17 smithi181 ceph-mon[106466]: failed to encode map e966 with expected crc 2024-03-29T05:10:17.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:17 smithi181 ceph-mon[106466]: failed to encode map e966 with expected crc 2024-03-29T05:10:17.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:17 smithi181 ceph-mon[106466]: failed to encode map e966 with expected crc 2024-03-29T05:10:17.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:17 smithi181 ceph-mon[106466]: failed to encode map e966 with expected crc 2024-03-29T05:10:17.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:17 smithi181 ceph-mon[106466]: osdmap e967: 8 total, 8 up, 8 in 2024-03-29T05:10:17.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:17 smithi181 ceph-mon[101970]: pgmap v1178: 105 pgs: 105 active+clean; 600 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 6.2 KiB/s wr, 196 op/s 2024-03-29T05:10:17.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:17 smithi181 ceph-mon[101970]: failed to encode map e966 with expected crc 2024-03-29T05:10:17.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:17 smithi181 ceph-mon[101970]: failed to encode map e966 with expected crc 2024-03-29T05:10:17.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:17 smithi181 ceph-mon[101970]: failed to encode map e966 with expected crc 2024-03-29T05:10:17.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:17 smithi181 ceph-mon[101970]: failed to encode map e966 with expected crc 2024-03-29T05:10:17.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:17 smithi181 ceph-mon[101970]: failed to encode map e966 with expected crc 2024-03-29T05:10:17.454 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:17 smithi181 ceph-mon[101970]: osdmap e967: 8 total, 8 up, 8 in 2024-03-29T05:10:17.854 INFO:tasks.workunit.client.0.smithi181.stdout: wrote [4194304~12582912] to clone 2024-03-29T05:10:17.855 INFO:tasks.workunit.client.0.smithi181.stdout: diff was [4194304~12582912] 2024-03-29T05:10:17.858 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/1.DiffIterateIgnoreParent (1003 ms) 2024-03-29T05:10:17.858 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/1.DiffIterateCallbackError 2024-03-29T05:10:17.858 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:10:17.879 INFO:tasks.workunit.client.0.smithi181.stdout: wrote [673161~35332,5776717~56038,10237502~55484,13749074~46126,13941468~88026,14559022~67015,17734658~950,19785778~39694,19833489~91016,20829373~39498] 2024-03-29T05:10:17.890 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/1.DiffIterateCallbackError (32 ms) 2024-03-29T05:10:17.890 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParentDiscard 2024-03-29T05:10:17.890 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:10:18.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:18 smithi187 ceph-mon[93536]: failed to encode map e967 with expected crc 2024-03-29T05:10:18.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:18 smithi187 ceph-mon[93536]: failed to encode map e967 with expected crc 2024-03-29T05:10:18.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:18 smithi187 ceph-mon[93536]: failed to encode map e967 with expected crc 2024-03-29T05:10:18.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:18 smithi187 ceph-mon[93536]: osdmap e968: 8 total, 8 up, 8 in 2024-03-29T05:10:18.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:18 smithi181 ceph-mon[101970]: failed to encode map e967 with expected crc 2024-03-29T05:10:18.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:18 smithi181 ceph-mon[101970]: failed to encode map e967 with expected crc 2024-03-29T05:10:18.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:18 smithi181 ceph-mon[101970]: failed to encode map e967 with expected crc 2024-03-29T05:10:18.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:18 smithi181 ceph-mon[101970]: osdmap e968: 8 total, 8 up, 8 in 2024-03-29T05:10:18.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:18 smithi181 ceph-mon[106466]: failed to encode map e967 with expected crc 2024-03-29T05:10:18.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:18 smithi181 ceph-mon[106466]: failed to encode map e967 with expected crc 2024-03-29T05:10:18.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:18 smithi181 ceph-mon[106466]: failed to encode map e967 with expected crc 2024-03-29T05:10:18.454 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:18 smithi181 ceph-mon[106466]: osdmap e968: 8 total, 8 up, 8 in 2024-03-29T05:10:19.203 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:10:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:10:18] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.43.0" 2024-03-29T05:10:19.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:19 smithi181 ceph-mon[106466]: pgmap v1181: 105 pgs: 105 active+clean; 531 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 3.0 MiB/s wr, 121 op/s 2024-03-29T05:10:19.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:19 smithi181 ceph-mon[106466]: failed to encode map e968 with expected crc 2024-03-29T05:10:19.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:19 smithi181 ceph-mon[106466]: failed to encode map e968 with expected crc 2024-03-29T05:10:19.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:19 smithi181 ceph-mon[106466]: osdmap e969: 8 total, 8 up, 8 in 2024-03-29T05:10:19.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:19 smithi181 ceph-mon[101970]: pgmap v1181: 105 pgs: 105 active+clean; 531 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 3.0 MiB/s wr, 121 op/s 2024-03-29T05:10:19.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:19 smithi181 ceph-mon[101970]: failed to encode map e968 with expected crc 2024-03-29T05:10:19.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:19 smithi181 ceph-mon[101970]: failed to encode map e968 with expected crc 2024-03-29T05:10:19.204 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:19 smithi181 ceph-mon[101970]: osdmap e969: 8 total, 8 up, 8 in 2024-03-29T05:10:19.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:19 smithi187 ceph-mon[93536]: pgmap v1181: 105 pgs: 105 active+clean; 531 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 3.0 MiB/s wr, 121 op/s 2024-03-29T05:10:19.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:19 smithi187 ceph-mon[93536]: failed to encode map e968 with expected crc 2024-03-29T05:10:19.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:19 smithi187 ceph-mon[93536]: failed to encode map e968 with expected crc 2024-03-29T05:10:19.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:19 smithi187 ceph-mon[93536]: osdmap e969: 8 total, 8 up, 8 in 2024-03-29T05:10:19.853 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/1.DiffIterateParentDiscard (1963 ms) 2024-03-29T05:10:19.853 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnalignedSmall 2024-03-29T05:10:19.853 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:10:19.914 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnalignedSmall (61 ms) 2024-03-29T05:10:19.914 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnaligned 2024-03-29T05:10:19.914 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:10:20.005 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnaligned (90 ms) 2024-03-29T05:10:20.005 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/1.DiffIterateTryAcquireLock 2024-03-29T05:10:20.005 INFO:tasks.workunit.client.0.smithi181.stdout:using new format! 2024-03-29T05:10:20.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:20 smithi187 ceph-mon[93536]: failed to encode map e969 with expected crc 2024-03-29T05:10:20.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:20 smithi187 ceph-mon[93536]: osdmap e970: 8 total, 8 up, 8 in 2024-03-29T05:10:20.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:20 smithi181 ceph-mon[101970]: failed to encode map e969 with expected crc 2024-03-29T05:10:20.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:20 smithi181 ceph-mon[101970]: osdmap e970: 8 total, 8 up, 8 in 2024-03-29T05:10:20.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:20 smithi181 ceph-mon[106466]: failed to encode map e969 with expected crc 2024-03-29T05:10:20.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:20 smithi181 ceph-mon[106466]: osdmap e970: 8 total, 8 up, 8 in 2024-03-29T05:10:21.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:21 smithi187 ceph-mon[93536]: failed to encode map e969 with expected crc 2024-03-29T05:10:21.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:21 smithi187 ceph-mon[93536]: pgmap v1184: 105 pgs: 105 active+clean; 476 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 113 KiB/s rd, 8.1 MiB/s wr, 163 op/s 2024-03-29T05:10:21.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:21 smithi187 ceph-mon[93536]: failed to encode map e970 with expected crc 2024-03-29T05:10:21.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:21 smithi181 ceph-mon[106466]: failed to encode map e969 with expected crc 2024-03-29T05:10:21.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:21 smithi181 ceph-mon[106466]: pgmap v1184: 105 pgs: 105 active+clean; 476 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 113 KiB/s rd, 8.1 MiB/s wr, 163 op/s 2024-03-29T05:10:21.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:21 smithi181 ceph-mon[106466]: failed to encode map e970 with expected crc 2024-03-29T05:10:21.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:21 smithi181 ceph-mon[101970]: failed to encode map e969 with expected crc 2024-03-29T05:10:21.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:21 smithi181 ceph-mon[101970]: pgmap v1184: 105 pgs: 105 active+clean; 476 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 113 KiB/s rd, 8.1 MiB/s wr, 163 op/s 2024-03-29T05:10:21.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:21 smithi181 ceph-mon[101970]: failed to encode map e970 with expected crc 2024-03-29T05:10:23.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:23 smithi187 ceph-mon[93536]: pgmap v1186: 105 pgs: 105 active+clean; 476 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 99 KiB/s rd, 4.7 MiB/s wr, 140 op/s 2024-03-29T05:10:23.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:23 smithi187 ceph-mon[93536]: osdmap e971: 8 total, 8 up, 8 in 2024-03-29T05:10:23.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:23 smithi181 ceph-mon[101970]: pgmap v1186: 105 pgs: 105 active+clean; 476 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 99 KiB/s rd, 4.7 MiB/s wr, 140 op/s 2024-03-29T05:10:23.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:23 smithi181 ceph-mon[101970]: osdmap e971: 8 total, 8 up, 8 in 2024-03-29T05:10:23.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:23 smithi181 ceph-mon[106466]: pgmap v1186: 105 pgs: 105 active+clean; 476 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 99 KiB/s rd, 4.7 MiB/s wr, 140 op/s 2024-03-29T05:10:23.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:23 smithi181 ceph-mon[106466]: osdmap e971: 8 total, 8 up, 8 in 2024-03-29T05:10:24.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:24 smithi187 ceph-mon[93536]: failed to encode map e971 with expected crc 2024-03-29T05:10:24.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:24 smithi187 ceph-mon[93536]: failed to encode map e971 with expected crc 2024-03-29T05:10:24.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:24 smithi187 ceph-mon[93536]: failed to encode map e971 with expected crc 2024-03-29T05:10:24.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:24 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:10:24.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:24 smithi181 ceph-mon[106466]: failed to encode map e971 with expected crc 2024-03-29T05:10:24.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:24 smithi181 ceph-mon[106466]: failed to encode map e971 with expected crc 2024-03-29T05:10:24.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:24 smithi181 ceph-mon[106466]: failed to encode map e971 with expected crc 2024-03-29T05:10:24.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:24 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:10:24.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:24 smithi181 ceph-mon[101970]: failed to encode map e971 with expected crc 2024-03-29T05:10:24.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:24 smithi181 ceph-mon[101970]: failed to encode map e971 with expected crc 2024-03-29T05:10:24.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:24 smithi181 ceph-mon[101970]: failed to encode map e971 with expected crc 2024-03-29T05:10:24.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:24 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:10:25.049 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/1.DiffIterateTryAcquireLock (5045 ms) 2024-03-29T05:10:25.049 INFO:tasks.workunit.client.0.smithi181.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStriping 2024-03-29T05:10:25.049 INFO:tasks.workunit.client.0.smithi181.stdout:SKIPPING 2024-03-29T05:10:25.049 INFO:tasks.workunit.client.0.smithi181.stdout:[ OK ] DiffIterateTest/1.DiffIterateStriping (0 ms) 2024-03-29T05:10:25.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:25 smithi187 ceph-mon[93536]: pgmap v1188: 105 pgs: 105 active+clean; 548 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 7.8 MiB/s wr, 240 op/s 2024-03-29T05:10:25.453 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:25 smithi181 ceph-mon[106466]: pgmap v1188: 105 pgs: 105 active+clean; 548 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 7.8 MiB/s wr, 240 op/s 2024-03-29T05:10:25.453 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:25 smithi181 ceph-mon[101970]: pgmap v1188: 105 pgs: 105 active+clean; 548 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 7.8 MiB/s wr, 240 op/s 2024-03-29T05:10:26.048 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] 15 tests from DiffIterateTest/1 (67666 ms total) 2024-03-29T05:10:26.048 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:10:26.048 INFO:tasks.workunit.client.0.smithi181.stdout:[----------] Global test environment tear-down 2024-03-29T05:10:26.049 INFO:tasks.workunit.client.0.smithi181.stdout:[==========] 415 tests from 16 test suites ran. (1034279 ms total) 2024-03-29T05:10:26.049 INFO:tasks.workunit.client.0.smithi181.stdout:[ PASSED ] 415 tests. 2024-03-29T05:10:26.049 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:10:26.049 INFO:tasks.workunit.client.0.smithi181.stdout: YOU HAVE 2 DISABLED TESTS 2024-03-29T05:10:26.049 INFO:tasks.workunit.client.0.smithi181.stdout: 2024-03-29T05:10:26.056 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-03-29T05:10:26.056 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-03-29T05:10:26.126 INFO:tasks.workunit:Stopping ['rbd/test_librbd.sh'] on client.0... 2024-03-29T05:10:26.127 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-03-29T05:10:26.396 DEBUG:teuthology.parallel:result is None 2024-03-29T05:10:26.396 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-03-29T05:10:26.423 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-03-29T05:10:26.423 DEBUG:teuthology.orchestra.run.smithi181:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-03-29T05:10:26.482 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-03-29T05:10:26.483 INFO:teuthology.task.sequential:In sequential, running task print... 2024-03-29T05:10:26.483 INFO:teuthology.task.print:**** done rbd/test_librbd.sh 7-workload 2024-03-29T05:10:26.483 DEBUG:teuthology.parallel:result is None 2024-03-29T05:10:26.483 INFO:teuthology.run_tasks:Running task print... 2024-03-29T05:10:26.493 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-03-29T05:10:26.493 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-03-29T05:10:26.533 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-29T05:10:26.534 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-29T05:10:26.534 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph mgr dump --format=json 2024-03-29T05:10:27.122 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:27.369 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:27 smithi181 ceph-mon[101970]: pgmap v1189: 105 pgs: 105 active+clean; 552 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 6.5 MiB/s wr, 180 op/s 2024-03-29T05:10:27.369 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:27 smithi181 ceph-mon[101970]: failed to encode map e972 with expected crc 2024-03-29T05:10:27.369 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:27 smithi181 ceph-mon[101970]: osdmap e972: 8 total, 8 up, 8 in 2024-03-29T05:10:27.369 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:27 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:10:27.369 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:27 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:10:27.369 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:27 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:10:27.369 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:27 smithi181 ceph-mon[106466]: pgmap v1189: 105 pgs: 105 active+clean; 552 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 6.5 MiB/s wr, 180 op/s 2024-03-29T05:10:27.369 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:27 smithi181 ceph-mon[106466]: failed to encode map e972 with expected crc 2024-03-29T05:10:27.369 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:27 smithi181 ceph-mon[106466]: osdmap e972: 8 total, 8 up, 8 in 2024-03-29T05:10:27.370 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:27 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:10:27.370 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:27 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:10:27.370 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:27 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:10:27.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:27 smithi187 ceph-mon[93536]: pgmap v1189: 105 pgs: 105 active+clean; 552 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 6.5 MiB/s wr, 180 op/s 2024-03-29T05:10:27.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:27 smithi187 ceph-mon[93536]: failed to encode map e972 with expected crc 2024-03-29T05:10:27.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:27 smithi187 ceph-mon[93536]: osdmap e972: 8 total, 8 up, 8 in 2024-03-29T05:10:27.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:27 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:10:27.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:27 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:10:27.383 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:27 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:10:28.614 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:28 smithi181 ceph-mon[101970]: pgmap v1191: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:10:28.614 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:28 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:10:28.614 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:28 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:10:28.614 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:28 smithi181 ceph-mon[106466]: pgmap v1191: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:10:28.614 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:28 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:10:28.614 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:28 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:10:28.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:28 smithi187 ceph-mon[93536]: pgmap v1191: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T05:10:28.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:28 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:10:28.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:28 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:10:29.101 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:10:29.276 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:10:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:10:28] "GET /metrics HTTP/1.1" 200 34117 "" "Prometheus/2.43.0" 2024-03-29T05:10:29.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:29 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:10:29.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:29 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:10:29.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:29 smithi187 ceph-mon[93536]: pgmap v1192: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:29.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:29 smithi187 ceph-mon[93536]: pgmap v1193: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:29.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:29 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:10:29.633 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:29 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3697702120' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T05:10:29.656 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:29 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:10:29.656 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:29 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:10:29.656 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:29 smithi181 ceph-mon[106466]: pgmap v1192: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:29.657 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:29 smithi181 ceph-mon[106466]: pgmap v1193: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:29.657 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:29 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:10:29.657 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:29 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3697702120' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T05:10:29.657 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:29 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:10:29.657 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:29 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:10:29.657 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:29 smithi181 ceph-mon[101970]: pgmap v1192: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:29.657 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:29 smithi181 ceph-mon[101970]: pgmap v1193: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:29.657 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:29 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:10:29.657 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:29 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3697702120' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T05:10:29.680 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":37,"flags":0,"active_gid":25337,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6800","nonce":1061296634},{"type":"v1","addr":"172.21.15.181:6801","nonce":1061296634}]},"active_addr":"172.21.15.181:6801/1061296634","active_change":"2024-03-29T04:56:11.126987+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":25153,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:latest","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:latest","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.181:8443/","prometheus":"http://172.21.15.181:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":245,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.181:0","nonce":1807651403}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.181:0","nonce":4038389440}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.181:0","nonce":2846786227}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.181:0","nonce":2762574662}]}]} 2024-03-29T05:10:29.684 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-29T05:10:29.684 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-29T05:10:29.684 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T05:10:29.938 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:31.702 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:10:31.702 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":972,"fsid":"dfcecab8-ed86-11ee-95d0-87774f69a715","created":"2024-03-29T04:44:49.014354+0000","modified":"2024-03-29T05:10:26.032143+0000","last_up_change":"2024-03-29T05:02:14.228937+0000","last_in_change":"2024-03-29T04:49:44.602034+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":41,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:48:01.551395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:50:37.400395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-29T04:51:49.771458+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-29T04:51:50.114853+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"26991441-a6cb-4694-aec3-0f793de6d551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":413,"up_from":425,"up_thru":908,"down_at":414,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":849347250},{"type":"v1","addr":"172.21.15.181:6803","nonce":849347250}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":849347250},{"type":"v1","addr":"172.21.15.181:6805","nonce":849347250}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":849347250},{"type":"v1","addr":"172.21.15.181:6809","nonce":849347250}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":849347250},{"type":"v1","addr":"172.21.15.181:6807","nonce":849347250}]},"public_addr":"172.21.15.181:6803/849347250","cluster_addr":"172.21.15.181:6805/849347250","heartbeat_back_addr":"172.21.15.181:6809/849347250","heartbeat_front_addr":"172.21.15.181:6807/849347250","state":["exists","up"]},{"osd":1,"uuid":"04b645ce-4eb8-43bc-9f28-e3be932f4b1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":479,"up_from":490,"up_thru":908,"down_at":480,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":3877140947},{"type":"v1","addr":"172.21.15.181:6811","nonce":3877140947}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":3877140947},{"type":"v1","addr":"172.21.15.181:6813","nonce":3877140947}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":3877140947},{"type":"v1","addr":"172.21.15.181:6817","nonce":3877140947}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":3877140947},{"type":"v1","addr":"172.21.15.181:6815","nonce":3877140947}]},"public_addr":"172.21.15.181:6811/3877140947","cluster_addr":"172.21.15.181:6813/3877140947","heartbeat_back_addr":"172.21.15.181:6817/3877140947","heartbeat_front_addr":"172.21.15.181:6815/3877140947","state":["exists","up"]},{"osd":2,"uuid":"f72d641c-5d86-4d3a-bf90-d99c1c46a805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":514,"up_from":525,"up_thru":908,"down_at":515,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1623440102},{"type":"v1","addr":"172.21.15.181:6819","nonce":1623440102}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1623440102},{"type":"v1","addr":"172.21.15.181:6821","nonce":1623440102}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":1623440102},{"type":"v1","addr":"172.21.15.181:6825","nonce":1623440102}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1623440102},{"type":"v1","addr":"172.21.15.181:6823","nonce":1623440102}]},"public_addr":"172.21.15.181:6819/1623440102","cluster_addr":"172.21.15.181:6821/1623440102","heartbeat_back_addr":"172.21.15.181:6825/1623440102","heartbeat_front_addr":"172.21.15.181:6823/1623440102","state":["exists","up"]},{"osd":3,"uuid":"63ebbc1a-43b5-4990-87aa-f8f0ff975830","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":544,"up_from":555,"up_thru":908,"down_at":545,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":949048273},{"type":"v1","addr":"172.21.15.181:6827","nonce":949048273}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":949048273},{"type":"v1","addr":"172.21.15.181:6829","nonce":949048273}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6832","nonce":949048273},{"type":"v1","addr":"172.21.15.181:6833","nonce":949048273}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":949048273},{"type":"v1","addr":"172.21.15.181:6831","nonce":949048273}]},"public_addr":"172.21.15.181:6827/949048273","cluster_addr":"172.21.15.181:6829/949048273","heartbeat_back_addr":"172.21.15.181:6833/949048273","heartbeat_front_addr":"172.21.15.181:6831/949048273","state":["exists","up"]},{"osd":4,"uuid":"0a71f228-2cc8-4b83-b86d-62a2854a8533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":565,"up_from":576,"up_thru":908,"down_at":566,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6800","nonce":1357613329},{"type":"v1","addr":"172.21.15.187:6801","nonce":1357613329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6802","nonce":1357613329},{"type":"v1","addr":"172.21.15.187:6803","nonce":1357613329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6806","nonce":1357613329},{"type":"v1","addr":"172.21.15.187:6807","nonce":1357613329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6804","nonce":1357613329},{"type":"v1","addr":"172.21.15.187:6805","nonce":1357613329}]},"public_addr":"172.21.15.187:6801/1357613329","cluster_addr":"172.21.15.187:6803/1357613329","heartbeat_back_addr":"172.21.15.187:6807/1357613329","heartbeat_front_addr":"172.21.15.187:6805/1357613329","state":["exists","up"]},{"osd":5,"uuid":"847657e4-7b9a-4eee-9df5-8e32d0d6543e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":908,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6808","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6809","nonce":989620923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6810","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6811","nonce":989620923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6814","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6815","nonce":989620923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6812","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6813","nonce":989620923}]},"public_addr":"172.21.15.187:6809/989620923","cluster_addr":"172.21.15.187:6811/989620923","heartbeat_back_addr":"172.21.15.187:6815/989620923","heartbeat_front_addr":"172.21.15.187:6813/989620923","state":["exists","up"]},{"osd":6,"uuid":"cead714e-ffc1-44a4-bef5-cfa772070c9f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":908,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6816","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6817","nonce":1708678714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6818","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6819","nonce":1708678714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6822","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6823","nonce":1708678714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6820","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6821","nonce":1708678714}]},"public_addr":"172.21.15.187:6817/1708678714","cluster_addr":"172.21.15.187:6819/1708678714","heartbeat_back_addr":"172.21.15.187:6823/1708678714","heartbeat_front_addr":"172.21.15.187:6821/1708678714","state":["exists","up"]},{"osd":7,"uuid":"9e2520cb-e4ba-4b2a-9206-8a250e7a373f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":908,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6824","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6825","nonce":2139604103}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6826","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6827","nonce":2139604103}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6830","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6831","nonce":2139604103}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6828","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6829","nonce":2139604103}]},"public_addr":"172.21.15.187:6825/2139604103","cluster_addr":"172.21.15.187:6827/2139604103","heartbeat_back_addr":"172.21.15.187:6831/2139604103","heartbeat_front_addr":"172.21.15.187:6829/2139604103","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:08.364426+0000","dead_epoch":412},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:33.120007+0000","dead_epoch":479},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:57.511900+0000","dead_epoch":514},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:48:22.235902+0000","dead_epoch":544},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:48:45.159568+0000","dead_epoch":565},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:49:10.475329+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:49:34.890974+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:50:00.243723+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.181:0/3116926796":"2024-03-29T06:01:34.714745+0000","172.21.15.181:0/2023235":"2024-03-29T06:01:33.715725+0000","172.21.15.181:0/2279022951":"2024-03-29T06:05:54.484138+0000","172.21.15.181:0/3143565304":"2024-03-30T04:56:11.126763+0000","172.21.15.181:0/1741596559":"2024-03-30T04:54:27.760829+0000","172.21.15.187:6834/1172072464":"2024-03-30T04:55:24.992325+0000","172.21.15.181:0/3491810134":"2024-03-30T04:50:33.576089+0000","172.21.15.181:6800/3044637144":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/4052605721":"2024-03-30T04:45:25.467994+0000","172.21.15.187:0/3154176481":"2024-03-30T04:55:24.992325+0000","172.21.15.181:0/1867140711":"2024-03-30T04:45:49.234812+0000","172.21.15.187:0/642560288":"2024-03-30T04:55:24.992325+0000","172.21.15.181:6800/2540916225":"2024-03-30T04:45:49.234812+0000","172.21.15.181:0/33304889":"2024-03-30T04:56:11.126763+0000","172.21.15.181:0/1220271702":"2024-03-30T04:54:27.760829+0000","172.21.15.181:0/196688700":"2024-03-30T04:45:25.467994+0000","172.21.15.181:6801/3044637144":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/2280393448":"2024-03-30T04:54:27.760829+0000","172.21.15.181:6800/2863027222":"2024-03-30T04:45:25.467994+0000","172.21.15.181:6801/2540916225":"2024-03-30T04:45:49.234812+0000","172.21.15.187:6835/1172072464":"2024-03-30T04:55:24.992325+0000","172.21.15.181:0/114113941":"2024-03-30T04:45:49.234812+0000","172.21.15.181:6801/2863027222":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/3860889501":"2024-03-30T04:54:27.760829+0000","172.21.15.181:0/3153434425":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/2056930738":"2024-03-30T04:45:25.467994+0000","172.21.15.181:6801/3281522413":"2024-03-30T04:54:27.760829+0000","172.21.15.181:0/1768629494":"2024-03-30T04:45:49.234812+0000","172.21.15.187:0/2688587430":"2024-03-30T04:55:24.992325+0000","172.21.15.181:0/4244016576":"2024-03-30T04:50:33.576089+0000","172.21.15.187:0/404866275":"2024-03-30T04:55:24.992325+0000","172.21.15.181:6800/3281522413":"2024-03-30T04:54:27.760829+0000","172.21.15.187:0/2697850605":"2024-03-30T04:55:24.992325+0000","172.21.15.187:0/1113853736":"2024-03-30T04:55:24.992325+0000","172.21.15.181:0/209003815":"2024-03-30T04:56:11.126763+0000","172.21.15.181:6800/1502817592":"2024-03-30T04:56:11.126763+0000","172.21.15.181:6801/1502817592":"2024-03-30T04:56:11.126763+0000","172.21.15.181:0/1782908836":"2024-03-30T04:56:11.126763+0000","172.21.15.181:0/19173868":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/3372813659":"2024-03-30T04:56:11.126763+0000","172.21.15.181:0/2407547262":"2024-03-30T04:56:11.126763+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":2,"length":1}]},{"pool":14,"snaps":[{"begin":3,"length":6}]},{"pool":16,"snaps":[{"begin":34,"length":4},{"begin":44,"length":4}]},{"pool":17,"snaps":[{"begin":17,"length":2},{"begin":22,"length":1}]},{"pool":18,"snaps":[{"begin":2,"length":1}]},{"pool":40,"snaps":[{"begin":20,"length":2}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T05:10:31.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:31 smithi187 ceph-mon[93536]: pgmap v1194: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:31.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:31 smithi181 ceph-mon[106466]: pgmap v1194: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:31.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:31 smithi181 ceph-mon[101970]: pgmap v1194: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:32.354 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-29T05:10:32.354 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T05:10:32.609 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:32.860 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:32 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1905771893' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T05:10:32.860 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:32 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1905771893' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T05:10:32.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:32 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1905771893' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T05:10:33.694 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:33 smithi181 ceph-mon[106466]: pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:33.694 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:33 smithi181 ceph-mon[101970]: pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:33.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:33 smithi187 ceph-mon[93536]: pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:34.460 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:10:34.460 INFO:teuthology.orchestra.run.smithi181.stdout:{"epoch":972,"fsid":"dfcecab8-ed86-11ee-95d0-87774f69a715","created":"2024-03-29T04:44:49.014354+0000","modified":"2024-03-29T05:10:26.032143+0000","last_up_change":"2024-03-29T05:02:14.228937+0000","last_in_change":"2024-03-29T04:49:44.602034+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":41,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:48:01.551395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:50:37.400395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-29T04:51:49.771458+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-29T04:51:50.114853+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"26991441-a6cb-4694-aec3-0f793de6d551","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":413,"up_from":425,"up_thru":908,"down_at":414,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6802","nonce":849347250},{"type":"v1","addr":"172.21.15.181:6803","nonce":849347250}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6804","nonce":849347250},{"type":"v1","addr":"172.21.15.181:6805","nonce":849347250}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6808","nonce":849347250},{"type":"v1","addr":"172.21.15.181:6809","nonce":849347250}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6806","nonce":849347250},{"type":"v1","addr":"172.21.15.181:6807","nonce":849347250}]},"public_addr":"172.21.15.181:6803/849347250","cluster_addr":"172.21.15.181:6805/849347250","heartbeat_back_addr":"172.21.15.181:6809/849347250","heartbeat_front_addr":"172.21.15.181:6807/849347250","state":["exists","up"]},{"osd":1,"uuid":"04b645ce-4eb8-43bc-9f28-e3be932f4b1e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":479,"up_from":490,"up_thru":908,"down_at":480,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6810","nonce":3877140947},{"type":"v1","addr":"172.21.15.181:6811","nonce":3877140947}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6812","nonce":3877140947},{"type":"v1","addr":"172.21.15.181:6813","nonce":3877140947}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6816","nonce":3877140947},{"type":"v1","addr":"172.21.15.181:6817","nonce":3877140947}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6814","nonce":3877140947},{"type":"v1","addr":"172.21.15.181:6815","nonce":3877140947}]},"public_addr":"172.21.15.181:6811/3877140947","cluster_addr":"172.21.15.181:6813/3877140947","heartbeat_back_addr":"172.21.15.181:6817/3877140947","heartbeat_front_addr":"172.21.15.181:6815/3877140947","state":["exists","up"]},{"osd":2,"uuid":"f72d641c-5d86-4d3a-bf90-d99c1c46a805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":514,"up_from":525,"up_thru":908,"down_at":515,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6818","nonce":1623440102},{"type":"v1","addr":"172.21.15.181:6819","nonce":1623440102}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6820","nonce":1623440102},{"type":"v1","addr":"172.21.15.181:6821","nonce":1623440102}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6824","nonce":1623440102},{"type":"v1","addr":"172.21.15.181:6825","nonce":1623440102}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6822","nonce":1623440102},{"type":"v1","addr":"172.21.15.181:6823","nonce":1623440102}]},"public_addr":"172.21.15.181:6819/1623440102","cluster_addr":"172.21.15.181:6821/1623440102","heartbeat_back_addr":"172.21.15.181:6825/1623440102","heartbeat_front_addr":"172.21.15.181:6823/1623440102","state":["exists","up"]},{"osd":3,"uuid":"63ebbc1a-43b5-4990-87aa-f8f0ff975830","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":544,"up_from":555,"up_thru":908,"down_at":545,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6826","nonce":949048273},{"type":"v1","addr":"172.21.15.181:6827","nonce":949048273}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6828","nonce":949048273},{"type":"v1","addr":"172.21.15.181:6829","nonce":949048273}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6832","nonce":949048273},{"type":"v1","addr":"172.21.15.181:6833","nonce":949048273}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.181:6830","nonce":949048273},{"type":"v1","addr":"172.21.15.181:6831","nonce":949048273}]},"public_addr":"172.21.15.181:6827/949048273","cluster_addr":"172.21.15.181:6829/949048273","heartbeat_back_addr":"172.21.15.181:6833/949048273","heartbeat_front_addr":"172.21.15.181:6831/949048273","state":["exists","up"]},{"osd":4,"uuid":"0a71f228-2cc8-4b83-b86d-62a2854a8533","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":565,"up_from":576,"up_thru":908,"down_at":566,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6800","nonce":1357613329},{"type":"v1","addr":"172.21.15.187:6801","nonce":1357613329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6802","nonce":1357613329},{"type":"v1","addr":"172.21.15.187:6803","nonce":1357613329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6806","nonce":1357613329},{"type":"v1","addr":"172.21.15.187:6807","nonce":1357613329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6804","nonce":1357613329},{"type":"v1","addr":"172.21.15.187:6805","nonce":1357613329}]},"public_addr":"172.21.15.187:6801/1357613329","cluster_addr":"172.21.15.187:6803/1357613329","heartbeat_back_addr":"172.21.15.187:6807/1357613329","heartbeat_front_addr":"172.21.15.187:6805/1357613329","state":["exists","up"]},{"osd":5,"uuid":"847657e4-7b9a-4eee-9df5-8e32d0d6543e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":908,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6808","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6809","nonce":989620923}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6810","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6811","nonce":989620923}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6814","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6815","nonce":989620923}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6812","nonce":989620923},{"type":"v1","addr":"172.21.15.187:6813","nonce":989620923}]},"public_addr":"172.21.15.187:6809/989620923","cluster_addr":"172.21.15.187:6811/989620923","heartbeat_back_addr":"172.21.15.187:6815/989620923","heartbeat_front_addr":"172.21.15.187:6813/989620923","state":["exists","up"]},{"osd":6,"uuid":"cead714e-ffc1-44a4-bef5-cfa772070c9f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":908,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6816","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6817","nonce":1708678714}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6818","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6819","nonce":1708678714}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6822","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6823","nonce":1708678714}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6820","nonce":1708678714},{"type":"v1","addr":"172.21.15.187:6821","nonce":1708678714}]},"public_addr":"172.21.15.187:6817/1708678714","cluster_addr":"172.21.15.187:6819/1708678714","heartbeat_back_addr":"172.21.15.187:6823/1708678714","heartbeat_front_addr":"172.21.15.187:6821/1708678714","state":["exists","up"]},{"osd":7,"uuid":"9e2520cb-e4ba-4b2a-9206-8a250e7a373f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":908,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6824","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6825","nonce":2139604103}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6826","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6827","nonce":2139604103}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6830","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6831","nonce":2139604103}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.187:6828","nonce":2139604103},{"type":"v1","addr":"172.21.15.187:6829","nonce":2139604103}]},"public_addr":"172.21.15.187:6825/2139604103","cluster_addr":"172.21.15.187:6827/2139604103","heartbeat_back_addr":"172.21.15.187:6831/2139604103","heartbeat_front_addr":"172.21.15.187:6829/2139604103","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:08.364426+0000","dead_epoch":412},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:33.120007+0000","dead_epoch":479},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:47:57.511900+0000","dead_epoch":514},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:48:22.235902+0000","dead_epoch":544},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:48:45.159568+0000","dead_epoch":565},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:49:10.475329+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:49:34.890974+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:50:00.243723+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.181:0/3116926796":"2024-03-29T06:01:34.714745+0000","172.21.15.181:0/2023235":"2024-03-29T06:01:33.715725+0000","172.21.15.181:0/2279022951":"2024-03-29T06:05:54.484138+0000","172.21.15.181:0/3143565304":"2024-03-30T04:56:11.126763+0000","172.21.15.181:6800/3044637144":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/33304889":"2024-03-30T04:56:11.126763+0000","172.21.15.181:6800/2540916225":"2024-03-30T04:45:49.234812+0000","172.21.15.181:0/3491810134":"2024-03-30T04:50:33.576089+0000","172.21.15.181:6800/1502817592":"2024-03-30T04:56:11.126763+0000","172.21.15.181:0/4052605721":"2024-03-30T04:45:25.467994+0000","172.21.15.187:0/3154176481":"2024-03-30T04:55:24.992325+0000","172.21.15.187:0/642560288":"2024-03-30T04:55:24.992325+0000","172.21.15.181:6801/3044637144":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/2280393448":"2024-03-30T04:54:27.760829+0000","172.21.15.187:0/1113853736":"2024-03-30T04:55:24.992325+0000","172.21.15.181:0/209003815":"2024-03-30T04:56:11.126763+0000","172.21.15.187:6834/1172072464":"2024-03-30T04:55:24.992325+0000","172.21.15.181:0/196688700":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/1220271702":"2024-03-30T04:54:27.760829+0000","172.21.15.181:6801/1502817592":"2024-03-30T04:56:11.126763+0000","172.21.15.181:6800/2863027222":"2024-03-30T04:45:25.467994+0000","172.21.15.181:6801/2540916225":"2024-03-30T04:45:49.234812+0000","172.21.15.187:6835/1172072464":"2024-03-30T04:55:24.992325+0000","172.21.15.181:0/1867140711":"2024-03-30T04:45:49.234812+0000","172.21.15.181:0/2407547262":"2024-03-30T04:56:11.126763+0000","172.21.15.181:0/1741596559":"2024-03-30T04:54:27.760829+0000","172.21.15.181:6801/2863027222":"2024-03-30T04:45:25.467994+0000","172.21.15.181:0/114113941":"2024-03-30T04:45:49.234812+0000","172.21.15.181:0/3860889501":"2024-03-30T04:54:27.760829+0000","172.21.15.181:0/3153434425":"2024-03-30T04:50:33.576089+0000","172.21.15.181:0/2056930738":"2024-03-30T04:45:25.467994+0000","172.21.15.181:6801/3281522413":"2024-03-30T04:54:27.760829+0000","172.21.15.181:0/1768629494":"2024-03-30T04:45:49.234812+0000","172.21.15.187:0/2688587430":"2024-03-30T04:55:24.992325+0000","172.21.15.181:0/4244016576":"2024-03-30T04:50:33.576089+0000","172.21.15.187:0/404866275":"2024-03-30T04:55:24.992325+0000","172.21.15.181:0/1782908836":"2024-03-30T04:56:11.126763+0000","172.21.15.181:0/3372813659":"2024-03-30T04:56:11.126763+0000","172.21.15.181:0/19173868":"2024-03-30T04:50:33.576089+0000","172.21.15.181:6800/3281522413":"2024-03-30T04:54:27.760829+0000","172.21.15.187:0/2697850605":"2024-03-30T04:55:24.992325+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":2,"length":1}]},{"pool":14,"snaps":[{"begin":3,"length":6}]},{"pool":16,"snaps":[{"begin":34,"length":4},{"begin":44,"length":4}]},{"pool":17,"snaps":[{"begin":17,"length":2},{"begin":22,"length":1}]},{"pool":18,"snaps":[{"begin":2,"length":1}]},{"pool":40,"snaps":[{"begin":20,"length":2}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T05:10:34.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:34 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1693541032' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T05:10:34.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:34 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1693541032' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T05:10:34.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:34 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1693541032' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T05:10:35.018 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-29T05:10:35.018 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-29T05:10:35.019 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-29T05:10:35.019 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-29T05:10:35.020 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-29T05:10:35.020 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-29T05:10:35.020 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-29T05:10:35.021 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-29T05:10:35.554 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:35.557 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:35.561 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:35.566 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:35.567 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:35.573 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:35.575 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:35.603 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:35.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:35 smithi187 ceph-mon[93536]: pgmap v1196: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:35.889 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:35 smithi181 ceph-mon[101970]: pgmap v1196: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:35.889 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:35 smithi181 ceph-mon[106466]: pgmap v1196: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:37.685 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:37 smithi181 ceph-mon[106466]: pgmap v1197: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:37.685 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:37 smithi181 ceph-mon[101970]: pgmap v1197: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:37.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:37 smithi187 ceph-mon[93536]: pgmap v1197: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:39.339 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:10:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:10:38] "GET /metrics HTTP/1.1" 200 34117 "" "Prometheus/2.43.0" 2024-03-29T05:10:39.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:39 smithi187 ceph-mon[93536]: pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:39.938 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:39 smithi181 ceph-mon[101970]: pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:39.938 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:39 smithi181 ceph-mon[106466]: pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:41.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:41 smithi181 ceph-mon[101970]: pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:41.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:10:41.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:41 smithi181 ceph-mon[106466]: pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:41.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:10:41.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:41 smithi187 ceph-mon[93536]: pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:41.883 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:10:43.033 INFO:teuthology.orchestra.run.smithi181.stdout:2104533975162 2024-03-29T05:10:43.033 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-29T05:10:43.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:43 smithi181 ceph-mon[101970]: pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:43.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:43 smithi181 ceph-mon[106466]: pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:43.728 INFO:teuthology.orchestra.run.smithi181.stdout:2383706849388 2024-03-29T05:10:43.729 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-29T05:10:44.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:43 smithi187 ceph-mon[93536]: pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:45.903 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:45 smithi181 ceph-mon[101970]: pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:45.904 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:45 smithi181 ceph-mon[106466]: pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:46.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:45 smithi187 ceph-mon[93536]: pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:47.429 INFO:teuthology.orchestra.run.smithi181.stdout:1825361100937 2024-03-29T05:10:47.430 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-29T05:10:47.668 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:47 smithi181 ceph-mon[106466]: pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:47.669 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:47 smithi181 ceph-mon[101970]: pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:48.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:47 smithi187 ceph-mon[93536]: pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:49.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:10:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:10:48] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.43.0" 2024-03-29T05:10:49.805 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:49 smithi181 ceph-mon[101970]: pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:49.805 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:49 smithi181 ceph-mon[106466]: pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:50.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:49 smithi187 ceph-mon[93536]: pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:50.339 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:50.357 INFO:teuthology.orchestra.run.smithi181.stdout:167503724805 2024-03-29T05:10:50.357 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-29T05:10:50.593 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:51.013 INFO:teuthology.orchestra.run.smithi181.stdout:236223201532 2024-03-29T05:10:51.014 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-29T05:10:51.429 INFO:teuthology.orchestra.run.smithi181.stdout:2254857830516 2024-03-29T05:10:51.429 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-29T05:10:51.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:51 smithi181 ceph-mon[106466]: pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:51.703 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:51 smithi181 ceph-mon[101970]: pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:51.841 INFO:teuthology.orchestra.run.smithi181.stdout:201863463170 2024-03-29T05:10:51.842 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-29T05:10:52.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:51 smithi187 ceph-mon[93536]: pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:52.160 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:52.977 INFO:teuthology.orchestra.run.smithi181.stdout:2473901162601 2024-03-29T05:10:52.977 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-29T05:10:53.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:53 smithi181 ceph-mon[101970]: pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:53.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:53 smithi181 ceph-mon[106466]: pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:54.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:53 smithi187 ceph-mon[93536]: pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:54.411 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:54.411 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:54.412 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:55.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:55 smithi181 ceph-mon[101970]: pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:55.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:55 smithi181 ceph-mon[106466]: pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:55.987 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:10:56.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:55 smithi187 ceph-mon[93536]: pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:56.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:10:56.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:10:57.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:10:57.761 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:57 smithi181 ceph-mon[106466]: pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:57.761 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:57 smithi181 ceph-mon[101970]: pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:58.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:57 smithi187 ceph-mon[93536]: pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:59.203 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:10:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:10:58] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.43.0" 2024-03-29T05:10:59.738 INFO:teuthology.orchestra.run.smithi181.stdout:2104533975165 2024-03-29T05:10:59.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:10:59 smithi181 ceph-mon[101970]: pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:10:59.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:10:59 smithi181 ceph-mon[106466]: pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:00.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:10:59 smithi187 ceph-mon[93536]: pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:00.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:00 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1672495703' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T05:11:00.703 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:00 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1672495703' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T05:11:00.959 INFO:teuthology.orchestra.run.smithi181.stdout:2383706849391 2024-03-29T05:11:01.034 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:00 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1672495703' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T05:11:01.185 INFO:teuthology.orchestra.run.smithi181.stdout:1825361100939 2024-03-29T05:11:01.936 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:01 smithi181 ceph-mon[101970]: pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:01.936 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:01 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1622066261' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T05:11:01.936 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:01 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1737291901' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T05:11:01.936 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:01 smithi181 ceph-mon[106466]: pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:01.936 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:01 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1622066261' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T05:11:01.936 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:01 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1737291901' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T05:11:02.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:01 smithi187 ceph-mon[93536]: pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:02.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:01 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1622066261' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T05:11:02.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:01 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1737291901' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T05:11:02.303 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:11:02.326 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2104533975162 got 2104533975165 for osd.1 2024-03-29T05:11:02.326 DEBUG:teuthology.parallel:result is None 2024-03-29T05:11:02.868 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2383706849388 got 2383706849391 for osd.3 2024-03-29T05:11:02.868 DEBUG:teuthology.parallel:result is None 2024-03-29T05:11:03.615 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1825361100937 got 1825361100939 for osd.0 2024-03-29T05:11:03.616 DEBUG:teuthology.parallel:result is None 2024-03-29T05:11:03.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:03 smithi181 ceph-mon[106466]: pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:03.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:03 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3907760149' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T05:11:03.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:03 smithi181 ceph-mon[101970]: pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:03.953 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:03 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3907760149' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T05:11:04.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:03 smithi187 ceph-mon[93536]: pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:04.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:03 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3907760149' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T05:11:05.143 INFO:teuthology.orchestra.run.smithi181.stdout:201863463173 2024-03-29T05:11:05.222 INFO:teuthology.orchestra.run.smithi181.stdout:236223201535 2024-03-29T05:11:05.888 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:05 smithi181 ceph-mon[106466]: pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:05.888 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:05 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1008888555' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T05:11:05.888 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:05 smithi181 ceph-mon[101970]: pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:05.888 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:05 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1008888555' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T05:11:05.914 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463170 got 201863463173 for osd.6 2024-03-29T05:11:05.914 DEBUG:teuthology.parallel:result is None 2024-03-29T05:11:06.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:05 smithi187 ceph-mon[93536]: pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:06.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:05 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1008888555' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T05:11:07.001 INFO:teuthology.orchestra.run.smithi181.stdout:2254857830519 2024-03-29T05:11:07.197 INFO:teuthology.orchestra.run.smithi181.stdout:167503724809 2024-03-29T05:11:07.838 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:07 smithi181 ceph-mon[101970]: pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:07.838 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:07 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2784479262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T05:11:07.838 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:07 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3956859652' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T05:11:07.838 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:07 smithi181 ceph-mon[106466]: pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:07.839 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:07 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2784479262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T05:11:07.839 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:07 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3956859652' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T05:11:07.861 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724805 got 167503724809 for osd.5 2024-03-29T05:11:07.861 DEBUG:teuthology.parallel:result is None 2024-03-29T05:11:08.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:07 smithi187 ceph-mon[93536]: pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:08.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:07 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2784479262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T05:11:08.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:07 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3956859652' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T05:11:08.290 INFO:teuthology.orchestra.run.smithi181.stdout:2473901162604 2024-03-29T05:11:08.499 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2254857830516 got 2254857830519 for osd.2 2024-03-29T05:11:08.499 DEBUG:teuthology.parallel:result is None 2024-03-29T05:11:08.813 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:08 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/4137922990' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T05:11:08.813 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:08 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/4137922990' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T05:11:09.079 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:11:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:11:08] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.43.0" 2024-03-29T05:11:09.103 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201532 got 236223201535 for osd.7 2024-03-29T05:11:09.103 DEBUG:teuthology.parallel:result is None 2024-03-29T05:11:09.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:08 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/4137922990' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T05:11:09.557 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2473901162601 got 2473901162604 for osd.4 2024-03-29T05:11:09.557 DEBUG:teuthology.parallel:result is None 2024-03-29T05:11:09.558 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-29T05:11:09.558 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T05:11:09.818 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:11:09.934 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:09 smithi181 ceph-mon[101970]: pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:09.934 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:09 smithi181 ceph-mon[106466]: pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:10.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:09 smithi187 ceph-mon[93536]: pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:11.606 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:11:11.607 INFO:teuthology.orchestra.run.smithi181.stderr:dumped all 2024-03-29T05:11:11.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:11 smithi181 ceph-mon[101970]: pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:11.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:11:11.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:11 smithi181 ceph-mon[106466]: pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:11.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:11:12.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:11 smithi187 ceph-mon[93536]: pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:12.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:11:12.299 INFO:teuthology.orchestra.run.smithi181.stdout:{"pg_ready":true,"pg_map":{"version":1214,"stamp":"2024-03-29T05:11:10.551571+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1255444,"kb_used_data":73012,"kb_used_omap":278,"kb_used_meta":1182057,"kb_avail":748705772,"statfs":{"total":767960285184,"available":766674710528,"internally_reserved":0,"allocated":74764288,"data_stored":53132373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":285271,"internal_metadata":1210426793},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002570"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":1298,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.045549+0000","last_change":"2024-03-29T05:02:15.260835+0000","last_active":"2024-03-29T05:10:26.045549+0000","last_peered":"2024-03-29T05:10:26.045549+0000","last_clean":"2024-03-29T05:10:26.045549+0000","last_became_active":"2024-03-29T05:02:15.260504+0000","last_became_peered":"2024-03-29T05:02:15.260504+0000","last_unstale":"2024-03-29T05:10:26.045549+0000","last_undegraded":"2024-03-29T05:10:26.045549+0000","last_fullsized":"2024-03-29T05:10:26.045549+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:49:46.828819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":1454,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.802346+0000","last_change":"2024-03-29T05:01:18.501548+0000","last_active":"2024-03-29T05:10:26.802346+0000","last_peered":"2024-03-29T05:10:26.802346+0000","last_clean":"2024-03-29T05:10:26.802346+0000","last_became_active":"2024-03-29T05:01:18.501159+0000","last_became_peered":"2024-03-29T05:01:18.501159+0000","last_unstale":"2024-03-29T05:10:26.802346+0000","last_undegraded":"2024-03-29T05:10:26.802346+0000","last_fullsized":"2024-03-29T05:10:26.802346+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:22:10.843117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1340,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.865715+0000","last_change":"2024-03-29T05:01:53.840366+0000","last_active":"2024-03-29T05:10:26.865715+0000","last_peered":"2024-03-29T05:10:26.865715+0000","last_clean":"2024-03-29T05:10:26.865715+0000","last_became_active":"2024-03-29T05:01:53.840030+0000","last_became_peered":"2024-03-29T05:01:53.840030+0000","last_unstale":"2024-03-29T05:10:26.865715+0000","last_undegraded":"2024-03-29T05:10:26.865715+0000","last_fullsized":"2024-03-29T05:10:26.865715+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:15:15.057546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":1326,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.865573+0000","last_change":"2024-03-29T05:01:53.839814+0000","last_active":"2024-03-29T05:10:26.865573+0000","last_peered":"2024-03-29T05:10:26.865573+0000","last_clean":"2024-03-29T05:10:26.865573+0000","last_became_active":"2024-03-29T05:01:53.839486+0000","last_became_peered":"2024-03-29T05:01:53.839486+0000","last_unstale":"2024-03-29T05:10:26.865573+0000","last_undegraded":"2024-03-29T05:10:26.865573+0000","last_fullsized":"2024-03-29T05:10:26.865573+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:53:01.662020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1338,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.045570+0000","last_change":"2024-03-29T05:02:15.261932+0000","last_active":"2024-03-29T05:10:26.045570+0000","last_peered":"2024-03-29T05:10:26.045570+0000","last_clean":"2024-03-29T05:10:26.045570+0000","last_became_active":"2024-03-29T05:02:15.261350+0000","last_became_peered":"2024-03-29T05:02:15.261350+0000","last_unstale":"2024-03-29T05:10:26.045570+0000","last_undegraded":"2024-03-29T05:10:26.045570+0000","last_fullsized":"2024-03-29T05:10:26.045570+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:56:31.965997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":928,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.731173+0000","last_change":"2024-03-29T05:02:15.258277+0000","last_active":"2024-03-29T05:10:22.731173+0000","last_peered":"2024-03-29T05:10:22.731173+0000","last_clean":"2024-03-29T05:10:22.731173+0000","last_became_active":"2024-03-29T05:02:15.257769+0000","last_became_peered":"2024-03-29T05:02:15.257769+0000","last_unstale":"2024-03-29T05:10:22.731173+0000","last_undegraded":"2024-03-29T05:10:22.731173+0000","last_fullsized":"2024-03-29T05:10:22.731173+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:37:01.504077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1338,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.044709+0000","last_change":"2024-03-29T05:02:15.264103+0000","last_active":"2024-03-29T05:10:26.044709+0000","last_peered":"2024-03-29T05:10:26.044709+0000","last_clean":"2024-03-29T05:10:26.044709+0000","last_became_active":"2024-03-29T05:02:15.258201+0000","last_became_peered":"2024-03-29T05:02:15.258201+0000","last_unstale":"2024-03-29T05:10:26.044709+0000","last_undegraded":"2024-03-29T05:10:26.044709+0000","last_fullsized":"2024-03-29T05:10:26.044709+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:03:15.410934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":945,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.730778+0000","last_change":"2024-03-29T05:02:15.262078+0000","last_active":"2024-03-29T05:10:22.730778+0000","last_peered":"2024-03-29T05:10:22.730778+0000","last_clean":"2024-03-29T05:10:22.730778+0000","last_became_active":"2024-03-29T05:02:15.261728+0000","last_became_peered":"2024-03-29T05:02:15.261728+0000","last_unstale":"2024-03-29T05:10:22.730778+0000","last_undegraded":"2024-03-29T05:10:22.730778+0000","last_fullsized":"2024-03-29T05:10:22.730778+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:20:28.781180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1362,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.038934+0000","last_change":"2024-03-29T05:01:53.832409+0000","last_active":"2024-03-29T05:10:26.038934+0000","last_peered":"2024-03-29T05:10:26.038934+0000","last_clean":"2024-03-29T05:10:26.038934+0000","last_became_active":"2024-03-29T05:01:53.832075+0000","last_became_peered":"2024-03-29T05:01:53.832075+0000","last_unstale":"2024-03-29T05:10:26.038934+0000","last_undegraded":"2024-03-29T05:10:26.038934+0000","last_fullsized":"2024-03-29T05:10:26.038934+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:27:38.179605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":1450,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.801642+0000","last_change":"2024-03-29T05:02:15.263522+0000","last_active":"2024-03-29T05:10:26.801642+0000","last_peered":"2024-03-29T05:10:26.801642+0000","last_clean":"2024-03-29T05:10:26.801642+0000","last_became_active":"2024-03-29T05:02:15.263094+0000","last_became_peered":"2024-03-29T05:02:15.263094+0000","last_unstale":"2024-03-29T05:10:26.801642+0000","last_undegraded":"2024-03-29T05:10:26.801642+0000","last_fullsized":"2024-03-29T05:10:26.801642+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:02:19.808048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1309,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.044263+0000","last_change":"2024-03-29T05:02:15.258718+0000","last_active":"2024-03-29T05:10:26.044263+0000","last_peered":"2024-03-29T05:10:26.044263+0000","last_clean":"2024-03-29T05:10:26.044263+0000","last_became_active":"2024-03-29T05:02:15.256411+0000","last_became_peered":"2024-03-29T05:02:15.256411+0000","last_unstale":"2024-03-29T05:10:26.044263+0000","last_undegraded":"2024-03-29T05:10:26.044263+0000","last_fullsized":"2024-03-29T05:10:26.044263+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:08:52.734498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":1460,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.801223+0000","last_change":"2024-03-29T05:02:15.259190+0000","last_active":"2024-03-29T05:10:26.801223+0000","last_peered":"2024-03-29T05:10:26.801223+0000","last_clean":"2024-03-29T05:10:26.801223+0000","last_became_active":"2024-03-29T05:02:15.258814+0000","last_became_peered":"2024-03-29T05:02:15.258814+0000","last_unstale":"2024-03-29T05:10:26.801223+0000","last_undegraded":"2024-03-29T05:10:26.801223+0000","last_fullsized":"2024-03-29T05:10:26.801223+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:21:54.355754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":927,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.805092+0000","last_change":"2024-03-29T05:01:18.509131+0000","last_active":"2024-03-29T05:10:22.805092+0000","last_peered":"2024-03-29T05:10:22.805092+0000","last_clean":"2024-03-29T05:10:22.805092+0000","last_became_active":"2024-03-29T05:01:18.508777+0000","last_became_peered":"2024-03-29T05:01:18.508777+0000","last_unstale":"2024-03-29T05:10:22.805092+0000","last_undegraded":"2024-03-29T05:10:22.805092+0000","last_fullsized":"2024-03-29T05:10:22.805092+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:33:48.034434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1307,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.864748+0000","last_change":"2024-03-29T05:01:53.838294+0000","last_active":"2024-03-29T05:10:26.864748+0000","last_peered":"2024-03-29T05:10:26.864748+0000","last_clean":"2024-03-29T05:10:26.864748+0000","last_became_active":"2024-03-29T05:01:53.834516+0000","last_became_peered":"2024-03-29T05:01:53.834516+0000","last_unstale":"2024-03-29T05:10:26.864748+0000","last_undegraded":"2024-03-29T05:10:26.864748+0000","last_fullsized":"2024-03-29T05:10:26.864748+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:02:39.369882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1398,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.085442+0000","last_change":"2024-03-29T05:02:15.252323+0000","last_active":"2024-03-29T05:10:26.085442+0000","last_peered":"2024-03-29T05:10:26.085442+0000","last_clean":"2024-03-29T05:10:26.085442+0000","last_became_active":"2024-03-29T05:02:15.251982+0000","last_became_peered":"2024-03-29T05:02:15.251982+0000","last_unstale":"2024-03-29T05:10:26.085442+0000","last_undegraded":"2024-03-29T05:10:26.085442+0000","last_fullsized":"2024-03-29T05:10:26.085442+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:54:20.450633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":1310,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.865753+0000","last_change":"2024-03-29T05:01:53.836394+0000","last_active":"2024-03-29T05:10:26.865753+0000","last_peered":"2024-03-29T05:10:26.865753+0000","last_clean":"2024-03-29T05:10:26.865753+0000","last_became_active":"2024-03-29T05:01:53.834444+0000","last_became_peered":"2024-03-29T05:01:53.834444+0000","last_unstale":"2024-03-29T05:10:26.865753+0000","last_undegraded":"2024-03-29T05:10:26.865753+0000","last_fullsized":"2024-03-29T05:10:26.865753+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:02:57.479683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1439,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.802147+0000","last_change":"2024-03-29T04:59:33.396754+0000","last_active":"2024-03-29T05:10:26.802147+0000","last_peered":"2024-03-29T05:10:26.802147+0000","last_clean":"2024-03-29T05:10:26.802147+0000","last_became_active":"2024-03-29T04:59:33.395620+0000","last_became_peered":"2024-03-29T04:59:33.395620+0000","last_unstale":"2024-03-29T05:10:26.802147+0000","last_undegraded":"2024-03-29T05:10:26.802147+0000","last_fullsized":"2024-03-29T05:10:26.802147+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:32:54.303599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1399,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.085001+0000","last_change":"2024-03-29T05:01:18.501816+0000","last_active":"2024-03-29T05:10:26.085001+0000","last_peered":"2024-03-29T05:10:26.085001+0000","last_clean":"2024-03-29T05:10:26.085001+0000","last_became_active":"2024-03-29T05:01:18.501353+0000","last_became_peered":"2024-03-29T05:01:18.501353+0000","last_unstale":"2024-03-29T05:10:26.085001+0000","last_undegraded":"2024-03-29T05:10:26.085001+0000","last_fullsized":"2024-03-29T05:10:26.085001+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:38:26.980925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":941,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.090830+0000","last_change":"2024-03-29T05:02:15.261144+0000","last_active":"2024-03-29T05:10:23.090830+0000","last_peered":"2024-03-29T05:10:23.090830+0000","last_clean":"2024-03-29T05:10:23.090830+0000","last_became_active":"2024-03-29T05:02:15.260781+0000","last_became_peered":"2024-03-29T05:02:15.260781+0000","last_unstale":"2024-03-29T05:10:23.090830+0000","last_undegraded":"2024-03-29T05:10:23.090830+0000","last_fullsized":"2024-03-29T05:10:23.090830+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:25:19.644980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1461,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.801918+0000","last_change":"2024-03-29T05:01:18.505730+0000","last_active":"2024-03-29T05:10:26.801918+0000","last_peered":"2024-03-29T05:10:26.801918+0000","last_clean":"2024-03-29T05:10:26.801918+0000","last_became_active":"2024-03-29T05:01:18.505403+0000","last_became_peered":"2024-03-29T05:01:18.505403+0000","last_unstale":"2024-03-29T05:10:26.801918+0000","last_undegraded":"2024-03-29T05:10:26.801918+0000","last_fullsized":"2024-03-29T05:10:26.801918+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:15:52.154347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1456,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.802479+0000","last_change":"2024-03-29T05:02:15.258159+0000","last_active":"2024-03-29T05:10:26.802479+0000","last_peered":"2024-03-29T05:10:26.802479+0000","last_clean":"2024-03-29T05:10:26.802479+0000","last_became_active":"2024-03-29T05:02:15.257780+0000","last_became_peered":"2024-03-29T05:02:15.257780+0000","last_unstale":"2024-03-29T05:10:26.802479+0000","last_undegraded":"2024-03-29T05:10:26.802479+0000","last_fullsized":"2024-03-29T05:10:26.802479+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:32:25.749061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":941,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.090022+0000","last_change":"2024-03-29T05:00:43.919799+0000","last_active":"2024-03-29T05:10:23.090022+0000","last_peered":"2024-03-29T05:10:23.090022+0000","last_clean":"2024-03-29T05:10:23.090022+0000","last_became_active":"2024-03-29T05:00:43.919485+0000","last_became_peered":"2024-03-29T05:00:43.919485+0000","last_unstale":"2024-03-29T05:10:23.090022+0000","last_undegraded":"2024-03-29T05:10:23.090022+0000","last_fullsized":"2024-03-29T05:10:23.090022+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:26:33.257813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":922,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.805039+0000","last_change":"2024-03-29T05:02:15.266302+0000","last_active":"2024-03-29T05:10:22.805039+0000","last_peered":"2024-03-29T05:10:22.805039+0000","last_clean":"2024-03-29T05:10:22.805039+0000","last_became_active":"2024-03-29T05:02:15.266015+0000","last_became_peered":"2024-03-29T05:02:15.266015+0000","last_unstale":"2024-03-29T05:10:22.805039+0000","last_undegraded":"2024-03-29T05:10:22.805039+0000","last_fullsized":"2024-03-29T05:10:22.805039+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:46:54.101383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1300,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.864719+0000","last_change":"2024-03-29T05:01:53.840657+0000","last_active":"2024-03-29T05:10:26.864719+0000","last_peered":"2024-03-29T05:10:26.864719+0000","last_clean":"2024-03-29T05:10:26.864719+0000","last_became_active":"2024-03-29T05:01:53.838284+0000","last_became_peered":"2024-03-29T05:01:53.838284+0000","last_unstale":"2024-03-29T05:10:26.864719+0000","last_undegraded":"2024-03-29T05:10:26.864719+0000","last_fullsized":"2024-03-29T05:10:26.864719+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:16:16.577458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"247'161","reported_seq":1301,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.090050+0000","last_change":"2024-03-29T04:59:33.398429+0000","last_active":"2024-03-29T05:10:23.090050+0000","last_peered":"2024-03-29T05:10:23.090050+0000","last_clean":"2024-03-29T05:10:23.090050+0000","last_became_active":"2024-03-29T04:59:33.397970+0000","last_became_peered":"2024-03-29T04:59:33.397970+0000","last_unstale":"2024-03-29T05:10:23.090050+0000","last_undegraded":"2024-03-29T05:10:23.090050+0000","last_fullsized":"2024-03-29T05:10:23.090050+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:48:01.601528+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:48:01.601528+0000","last_clean_scrub_stamp":"2024-03-29T04:48:01.601528+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:42:56.518950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":930,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.730065+0000","last_change":"2024-03-29T05:01:18.506639+0000","last_active":"2024-03-29T05:10:22.730065+0000","last_peered":"2024-03-29T05:10:22.730065+0000","last_clean":"2024-03-29T05:10:22.730065+0000","last_became_active":"2024-03-29T05:01:18.506276+0000","last_became_peered":"2024-03-29T05:01:18.506276+0000","last_unstale":"2024-03-29T05:10:22.730065+0000","last_undegraded":"2024-03-29T05:10:22.730065+0000","last_fullsized":"2024-03-29T05:10:22.730065+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:10:41.303709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1396,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.086255+0000","last_change":"2024-03-29T05:00:43.908190+0000","last_active":"2024-03-29T05:10:26.086255+0000","last_peered":"2024-03-29T05:10:26.086255+0000","last_clean":"2024-03-29T05:10:26.086255+0000","last_became_active":"2024-03-29T05:00:43.907883+0000","last_became_peered":"2024-03-29T05:00:43.907883+0000","last_unstale":"2024-03-29T05:10:26.086255+0000","last_undegraded":"2024-03-29T05:10:26.086255+0000","last_fullsized":"2024-03-29T05:10:26.086255+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:31:27.009072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1399,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.084741+0000","last_change":"2024-03-29T05:01:18.509368+0000","last_active":"2024-03-29T05:10:26.084741+0000","last_peered":"2024-03-29T05:10:26.084741+0000","last_clean":"2024-03-29T05:10:26.084741+0000","last_became_active":"2024-03-29T05:01:18.508981+0000","last_became_peered":"2024-03-29T05:01:18.508981+0000","last_unstale":"2024-03-29T05:10:26.084741+0000","last_undegraded":"2024-03-29T05:10:26.084741+0000","last_fullsized":"2024-03-29T05:10:26.084741+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:33:33.633755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1386,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.038752+0000","last_change":"2024-03-29T05:01:53.832886+0000","last_active":"2024-03-29T05:10:26.038752+0000","last_peered":"2024-03-29T05:10:26.038752+0000","last_clean":"2024-03-29T05:10:26.038752+0000","last_became_active":"2024-03-29T05:01:53.832568+0000","last_became_peered":"2024-03-29T05:01:53.832568+0000","last_unstale":"2024-03-29T05:10:26.038752+0000","last_undegraded":"2024-03-29T05:10:26.038752+0000","last_fullsized":"2024-03-29T05:10:26.038752+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:20:06.656000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":939,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.730470+0000","last_change":"2024-03-29T05:01:53.824832+0000","last_active":"2024-03-29T05:10:22.730470+0000","last_peered":"2024-03-29T05:10:22.730470+0000","last_clean":"2024-03-29T05:10:22.730470+0000","last_became_active":"2024-03-29T05:01:53.824486+0000","last_became_peered":"2024-03-29T05:01:53.824486+0000","last_unstale":"2024-03-29T05:10:22.730470+0000","last_undegraded":"2024-03-29T05:10:22.730470+0000","last_fullsized":"2024-03-29T05:10:22.730470+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:23:48.429185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1317,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.045660+0000","last_change":"2024-03-29T05:02:15.264450+0000","last_active":"2024-03-29T05:10:26.045660+0000","last_peered":"2024-03-29T05:10:26.045660+0000","last_clean":"2024-03-29T05:10:26.045660+0000","last_became_active":"2024-03-29T05:02:15.261877+0000","last_became_peered":"2024-03-29T05:02:15.261877+0000","last_unstale":"2024-03-29T05:10:26.045660+0000","last_undegraded":"2024-03-29T05:10:26.045660+0000","last_fullsized":"2024-03-29T05:10:26.045660+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:55:23.320598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":930,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.730065+0000","last_change":"2024-03-29T05:00:43.922404+0000","last_active":"2024-03-29T05:10:22.730065+0000","last_peered":"2024-03-29T05:10:22.730065+0000","last_clean":"2024-03-29T05:10:22.730065+0000","last_became_active":"2024-03-29T05:00:43.922095+0000","last_became_peered":"2024-03-29T05:00:43.922095+0000","last_unstale":"2024-03-29T05:10:22.730065+0000","last_undegraded":"2024-03-29T05:10:22.730065+0000","last_fullsized":"2024-03-29T05:10:22.730065+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:34:54.348996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1399,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.085022+0000","last_change":"2024-03-29T05:02:15.249516+0000","last_active":"2024-03-29T05:10:26.085022+0000","last_peered":"2024-03-29T05:10:26.085022+0000","last_clean":"2024-03-29T05:10:26.085022+0000","last_became_active":"2024-03-29T05:02:15.249143+0000","last_became_peered":"2024-03-29T05:02:15.249143+0000","last_unstale":"2024-03-29T05:10:26.085022+0000","last_undegraded":"2024-03-29T05:10:26.085022+0000","last_fullsized":"2024-03-29T05:10:26.085022+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:26:36.986495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":943,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.731091+0000","last_change":"2024-03-29T05:01:53.826633+0000","last_active":"2024-03-29T05:10:22.731091+0000","last_peered":"2024-03-29T05:10:22.731091+0000","last_clean":"2024-03-29T05:10:22.731091+0000","last_became_active":"2024-03-29T05:01:53.826358+0000","last_became_peered":"2024-03-29T05:01:53.826358+0000","last_unstale":"2024-03-29T05:10:22.731091+0000","last_undegraded":"2024-03-29T05:10:22.731091+0000","last_fullsized":"2024-03-29T05:10:22.731091+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:04:56.270939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1402,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.085362+0000","last_change":"2024-03-29T05:00:43.910959+0000","last_active":"2024-03-29T05:10:26.085362+0000","last_peered":"2024-03-29T05:10:26.085362+0000","last_clean":"2024-03-29T05:10:26.085362+0000","last_became_active":"2024-03-29T05:00:43.909800+0000","last_became_peered":"2024-03-29T05:00:43.909800+0000","last_unstale":"2024-03-29T05:10:26.085362+0000","last_undegraded":"2024-03-29T05:10:26.085362+0000","last_fullsized":"2024-03-29T05:10:26.085362+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:37:21.817383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1407,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.086161+0000","last_change":"2024-03-29T05:02:15.249702+0000","last_active":"2024-03-29T05:10:26.086161+0000","last_peered":"2024-03-29T05:10:26.086161+0000","last_clean":"2024-03-29T05:10:26.086161+0000","last_became_active":"2024-03-29T05:02:15.249389+0000","last_became_peered":"2024-03-29T05:02:15.249389+0000","last_unstale":"2024-03-29T05:10:26.086161+0000","last_undegraded":"2024-03-29T05:10:26.086161+0000","last_fullsized":"2024-03-29T05:10:26.086161+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:21:44.015645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1332,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.864795+0000","last_change":"2024-03-29T05:01:53.839868+0000","last_active":"2024-03-29T05:10:26.864795+0000","last_peered":"2024-03-29T05:10:26.864795+0000","last_clean":"2024-03-29T05:10:26.864795+0000","last_became_active":"2024-03-29T05:01:53.837259+0000","last_became_peered":"2024-03-29T05:01:53.837259+0000","last_unstale":"2024-03-29T05:10:26.864795+0000","last_undegraded":"2024-03-29T05:10:26.864795+0000","last_fullsized":"2024-03-29T05:10:26.864795+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:49:29.224580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1298,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.045704+0000","last_change":"2024-03-29T05:02:15.258417+0000","last_active":"2024-03-29T05:10:26.045704+0000","last_peered":"2024-03-29T05:10:26.045704+0000","last_clean":"2024-03-29T05:10:26.045704+0000","last_became_active":"2024-03-29T05:02:15.257620+0000","last_became_peered":"2024-03-29T05:02:15.257620+0000","last_unstale":"2024-03-29T05:10:26.045704+0000","last_undegraded":"2024-03-29T05:10:26.045704+0000","last_fullsized":"2024-03-29T05:10:26.045704+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:58:46.529609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1293,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.044347+0000","last_change":"2024-03-29T05:02:15.256760+0000","last_active":"2024-03-29T05:10:26.044347+0000","last_peered":"2024-03-29T05:10:26.044347+0000","last_clean":"2024-03-29T05:10:26.044347+0000","last_became_active":"2024-03-29T05:02:15.255198+0000","last_became_peered":"2024-03-29T05:02:15.255198+0000","last_unstale":"2024-03-29T05:10:26.044347+0000","last_undegraded":"2024-03-29T05:10:26.044347+0000","last_fullsized":"2024-03-29T05:10:26.044347+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:26:40.696773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1308,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.044433+0000","last_change":"2024-03-29T05:02:15.256138+0000","last_active":"2024-03-29T05:10:26.044433+0000","last_peered":"2024-03-29T05:10:26.044433+0000","last_clean":"2024-03-29T05:10:26.044433+0000","last_became_active":"2024-03-29T05:02:15.254255+0000","last_became_peered":"2024-03-29T05:02:15.254255+0000","last_unstale":"2024-03-29T05:10:26.044433+0000","last_undegraded":"2024-03-29T05:10:26.044433+0000","last_fullsized":"2024-03-29T05:10:26.044433+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:45:49.901834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":940,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.805347+0000","last_change":"2024-03-29T05:02:15.268430+0000","last_active":"2024-03-29T05:10:22.805347+0000","last_peered":"2024-03-29T05:10:22.805347+0000","last_clean":"2024-03-29T05:10:22.805347+0000","last_became_active":"2024-03-29T05:02:15.268113+0000","last_became_peered":"2024-03-29T05:02:15.268113+0000","last_unstale":"2024-03-29T05:10:22.805347+0000","last_undegraded":"2024-03-29T05:10:22.805347+0000","last_fullsized":"2024-03-29T05:10:22.805347+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:48:31.192534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1320,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.044611+0000","last_change":"2024-03-29T05:02:15.261131+0000","last_active":"2024-03-29T05:10:26.044611+0000","last_peered":"2024-03-29T05:10:26.044611+0000","last_clean":"2024-03-29T05:10:26.044611+0000","last_became_active":"2024-03-29T05:02:15.260379+0000","last_became_peered":"2024-03-29T05:02:15.260379+0000","last_unstale":"2024-03-29T05:10:26.044611+0000","last_undegraded":"2024-03-29T05:10:26.044611+0000","last_fullsized":"2024-03-29T05:10:26.044611+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:27:11.830847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":1319,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.866207+0000","last_change":"2024-03-29T05:02:15.266435+0000","last_active":"2024-03-29T05:10:26.866207+0000","last_peered":"2024-03-29T05:10:26.866207+0000","last_clean":"2024-03-29T05:10:26.866207+0000","last_became_active":"2024-03-29T05:02:15.266112+0000","last_became_peered":"2024-03-29T05:02:15.266112+0000","last_unstale":"2024-03-29T05:10:26.866207+0000","last_undegraded":"2024-03-29T05:10:26.866207+0000","last_fullsized":"2024-03-29T05:10:26.866207+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:09:33.391187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1457,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.801846+0000","last_change":"2024-03-29T05:02:15.265027+0000","last_active":"2024-03-29T05:10:26.801846+0000","last_peered":"2024-03-29T05:10:26.801846+0000","last_clean":"2024-03-29T05:10:26.801846+0000","last_became_active":"2024-03-29T05:02:15.264664+0000","last_became_peered":"2024-03-29T05:02:15.264664+0000","last_unstale":"2024-03-29T05:10:26.801846+0000","last_undegraded":"2024-03-29T05:10:26.801846+0000","last_fullsized":"2024-03-29T05:10:26.801846+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:54:41.748257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":926,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.730643+0000","last_change":"2024-03-29T05:01:53.824320+0000","last_active":"2024-03-29T05:10:22.730643+0000","last_peered":"2024-03-29T05:10:22.730643+0000","last_clean":"2024-03-29T05:10:22.730643+0000","last_became_active":"2024-03-29T05:01:53.824010+0000","last_became_peered":"2024-03-29T05:01:53.824010+0000","last_unstale":"2024-03-29T05:10:22.730643+0000","last_undegraded":"2024-03-29T05:10:22.730643+0000","last_fullsized":"2024-03-29T05:10:22.730643+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:20:59.972126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":922,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.805214+0000","last_change":"2024-03-29T05:00:43.916507+0000","last_active":"2024-03-29T05:10:22.805214+0000","last_peered":"2024-03-29T05:10:22.805214+0000","last_clean":"2024-03-29T05:10:22.805214+0000","last_became_active":"2024-03-29T05:00:43.916027+0000","last_became_peered":"2024-03-29T05:00:43.916027+0000","last_unstale":"2024-03-29T05:10:22.805214+0000","last_undegraded":"2024-03-29T05:10:22.805214+0000","last_fullsized":"2024-03-29T05:10:22.805214+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:12:04.043477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":904,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.089792+0000","last_change":"2024-03-29T04:51:50.896476+0000","last_active":"2024-03-29T05:10:23.089792+0000","last_peered":"2024-03-29T05:10:23.089792+0000","last_clean":"2024-03-29T05:10:23.089792+0000","last_became_active":"2024-03-29T04:51:50.895258+0000","last_became_peered":"2024-03-29T04:51:50.895258+0000","last_unstale":"2024-03-29T05:10:23.089792+0000","last_undegraded":"2024-03-29T05:10:23.089792+0000","last_fullsized":"2024-03-29T05:10:23.089792+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:24:42.583134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1339,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.044132+0000","last_change":"2024-03-29T05:02:15.257042+0000","last_active":"2024-03-29T05:10:26.044132+0000","last_peered":"2024-03-29T05:10:26.044132+0000","last_clean":"2024-03-29T05:10:26.044132+0000","last_became_active":"2024-03-29T05:02:15.255472+0000","last_became_peered":"2024-03-29T05:02:15.255472+0000","last_unstale":"2024-03-29T05:10:26.044132+0000","last_undegraded":"2024-03-29T05:10:26.044132+0000","last_fullsized":"2024-03-29T05:10:26.044132+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:34:22.131400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":939,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.090843+0000","last_change":"2024-03-29T05:02:15.260429+0000","last_active":"2024-03-29T05:10:23.090843+0000","last_peered":"2024-03-29T05:10:23.090843+0000","last_clean":"2024-03-29T05:10:23.090843+0000","last_became_active":"2024-03-29T05:02:15.259772+0000","last_became_peered":"2024-03-29T05:02:15.259772+0000","last_unstale":"2024-03-29T05:10:23.090843+0000","last_undegraded":"2024-03-29T05:10:23.090843+0000","last_fullsized":"2024-03-29T05:10:23.090843+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:16:01.290463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":908,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.730407+0000","last_change":"2024-03-29T04:51:51.764872+0000","last_active":"2024-03-29T05:10:22.730407+0000","last_peered":"2024-03-29T05:10:22.730407+0000","last_clean":"2024-03-29T05:10:22.730407+0000","last_became_active":"2024-03-29T04:51:51.764554+0000","last_became_peered":"2024-03-29T04:51:51.764554+0000","last_unstale":"2024-03-29T05:10:22.730407+0000","last_undegraded":"2024-03-29T05:10:22.730407+0000","last_fullsized":"2024-03-29T05:10:22.730407+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:09:07.383326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":938,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.090913+0000","last_change":"2024-03-29T05:02:15.260665+0000","last_active":"2024-03-29T05:10:23.090913+0000","last_peered":"2024-03-29T05:10:23.090913+0000","last_clean":"2024-03-29T05:10:23.090913+0000","last_became_active":"2024-03-29T05:02:15.260149+0000","last_became_peered":"2024-03-29T05:02:15.260149+0000","last_unstale":"2024-03-29T05:10:23.090913+0000","last_undegraded":"2024-03-29T05:10:23.090913+0000","last_fullsized":"2024-03-29T05:10:23.090913+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:39:03.132413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1330,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.865624+0000","last_change":"2024-03-29T05:01:53.830781+0000","last_active":"2024-03-29T05:10:26.865624+0000","last_peered":"2024-03-29T05:10:26.865624+0000","last_clean":"2024-03-29T05:10:26.865624+0000","last_became_active":"2024-03-29T05:01:53.830556+0000","last_became_peered":"2024-03-29T05:01:53.830556+0000","last_unstale":"2024-03-29T05:10:26.865624+0000","last_undegraded":"2024-03-29T05:10:26.865624+0000","last_fullsized":"2024-03-29T05:10:26.865624+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:39:31.307835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":931,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.804572+0000","last_change":"2024-03-29T04:59:33.401536+0000","last_active":"2024-03-29T05:10:22.804572+0000","last_peered":"2024-03-29T05:10:22.804572+0000","last_clean":"2024-03-29T05:10:22.804572+0000","last_became_active":"2024-03-29T04:59:33.401119+0000","last_became_peered":"2024-03-29T04:59:33.401119+0000","last_unstale":"2024-03-29T05:10:22.804572+0000","last_undegraded":"2024-03-29T05:10:22.804572+0000","last_fullsized":"2024-03-29T05:10:22.804572+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:38:47.459663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1447,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.802291+0000","last_change":"2024-03-29T05:01:53.826079+0000","last_active":"2024-03-29T05:10:26.802291+0000","last_peered":"2024-03-29T05:10:26.802291+0000","last_clean":"2024-03-29T05:10:26.802291+0000","last_became_active":"2024-03-29T05:01:53.825729+0000","last_became_peered":"2024-03-29T05:01:53.825729+0000","last_unstale":"2024-03-29T05:10:26.802291+0000","last_undegraded":"2024-03-29T05:10:26.802291+0000","last_fullsized":"2024-03-29T05:10:26.802291+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:08:51.102523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":920,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.090140+0000","last_change":"2024-03-29T05:02:15.266708+0000","last_active":"2024-03-29T05:10:23.090140+0000","last_peered":"2024-03-29T05:10:23.090140+0000","last_clean":"2024-03-29T05:10:23.090140+0000","last_became_active":"2024-03-29T05:02:15.266130+0000","last_became_peered":"2024-03-29T05:02:15.266130+0000","last_unstale":"2024-03-29T05:10:23.090140+0000","last_undegraded":"2024-03-29T05:10:23.090140+0000","last_fullsized":"2024-03-29T05:10:23.090140+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:45:27.270692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":925,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.731071+0000","last_change":"2024-03-29T05:01:53.826799+0000","last_active":"2024-03-29T05:10:22.731071+0000","last_peered":"2024-03-29T05:10:22.731071+0000","last_clean":"2024-03-29T05:10:22.731071+0000","last_became_active":"2024-03-29T05:01:53.826480+0000","last_became_peered":"2024-03-29T05:01:53.826480+0000","last_unstale":"2024-03-29T05:10:22.731071+0000","last_undegraded":"2024-03-29T05:10:22.731071+0000","last_fullsized":"2024-03-29T05:10:22.731071+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:41:19.898891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1446,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.801511+0000","last_change":"2024-03-29T05:01:53.825119+0000","last_active":"2024-03-29T05:10:26.801511+0000","last_peered":"2024-03-29T05:10:26.801511+0000","last_clean":"2024-03-29T05:10:26.801511+0000","last_became_active":"2024-03-29T05:01:53.824758+0000","last_became_peered":"2024-03-29T05:01:53.824758+0000","last_unstale":"2024-03-29T05:10:26.801511+0000","last_undegraded":"2024-03-29T05:10:26.801511+0000","last_fullsized":"2024-03-29T05:10:26.801511+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:10:23.329902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1330,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.865211+0000","last_change":"2024-03-29T05:01:53.829740+0000","last_active":"2024-03-29T05:10:26.865211+0000","last_peered":"2024-03-29T05:10:26.865211+0000","last_clean":"2024-03-29T05:10:26.865211+0000","last_became_active":"2024-03-29T05:01:53.829454+0000","last_became_peered":"2024-03-29T05:01:53.829454+0000","last_unstale":"2024-03-29T05:10:26.865211+0000","last_undegraded":"2024-03-29T05:10:26.865211+0000","last_fullsized":"2024-03-29T05:10:26.865211+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:14:31.335348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":939,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.090418+0000","last_change":"2024-03-29T05:02:15.266831+0000","last_active":"2024-03-29T05:10:23.090418+0000","last_peered":"2024-03-29T05:10:23.090418+0000","last_clean":"2024-03-29T05:10:23.090418+0000","last_became_active":"2024-03-29T05:02:15.266462+0000","last_became_peered":"2024-03-29T05:02:15.266462+0000","last_unstale":"2024-03-29T05:10:23.090418+0000","last_undegraded":"2024-03-29T05:10:23.090418+0000","last_fullsized":"2024-03-29T05:10:23.090418+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:26:10.471305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1298,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.045436+0000","last_change":"2024-03-29T05:02:15.264444+0000","last_active":"2024-03-29T05:10:26.045436+0000","last_peered":"2024-03-29T05:10:26.045436+0000","last_clean":"2024-03-29T05:10:26.045436+0000","last_became_active":"2024-03-29T05:02:15.260792+0000","last_became_peered":"2024-03-29T05:02:15.260792+0000","last_unstale":"2024-03-29T05:10:26.045436+0000","last_undegraded":"2024-03-29T05:10:26.045436+0000","last_fullsized":"2024-03-29T05:10:26.045436+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:16:34.226178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"329'22","reported_seq":1305,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.044574+0000","last_change":"2024-03-29T05:02:15.262266+0000","last_active":"2024-03-29T05:10:26.044574+0000","last_peered":"2024-03-29T05:10:26.044574+0000","last_clean":"2024-03-29T05:10:26.044574+0000","last_became_active":"2024-03-29T05:02:15.261863+0000","last_became_peered":"2024-03-29T05:02:15.261863+0000","last_unstale":"2024-03-29T05:10:26.044574+0000","last_undegraded":"2024-03-29T05:10:26.044574+0000","last_fullsized":"2024-03-29T05:10:26.044574+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:08:24.941726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1398,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.084671+0000","last_change":"2024-03-29T05:01:18.509387+0000","last_active":"2024-03-29T05:10:26.084671+0000","last_peered":"2024-03-29T05:10:26.084671+0000","last_clean":"2024-03-29T05:10:26.084671+0000","last_became_active":"2024-03-29T05:01:18.509063+0000","last_became_peered":"2024-03-29T05:01:18.509063+0000","last_unstale":"2024-03-29T05:10:26.084671+0000","last_undegraded":"2024-03-29T05:10:26.084671+0000","last_fullsized":"2024-03-29T05:10:26.084671+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:31:04.765878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":940,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.090782+0000","last_change":"2024-03-29T05:02:15.260968+0000","last_active":"2024-03-29T05:10:23.090782+0000","last_peered":"2024-03-29T05:10:23.090782+0000","last_clean":"2024-03-29T05:10:23.090782+0000","last_became_active":"2024-03-29T05:02:15.260316+0000","last_became_peered":"2024-03-29T05:02:15.260316+0000","last_unstale":"2024-03-29T05:10:23.090782+0000","last_undegraded":"2024-03-29T05:10:23.090782+0000","last_fullsized":"2024-03-29T05:10:23.090782+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:46:24.566561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1300,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.865545+0000","last_change":"2024-03-29T05:01:53.843008+0000","last_active":"2024-03-29T05:10:26.865545+0000","last_peered":"2024-03-29T05:10:26.865545+0000","last_clean":"2024-03-29T05:10:26.865545+0000","last_became_active":"2024-03-29T05:01:53.842627+0000","last_became_peered":"2024-03-29T05:01:53.842627+0000","last_unstale":"2024-03-29T05:10:26.865545+0000","last_undegraded":"2024-03-29T05:10:26.865545+0000","last_fullsized":"2024-03-29T05:10:26.865545+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:54:18.788660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":926,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.730729+0000","last_change":"2024-03-29T05:00:43.914390+0000","last_active":"2024-03-29T05:10:22.730729+0000","last_peered":"2024-03-29T05:10:22.730729+0000","last_clean":"2024-03-29T05:10:22.730729+0000","last_became_active":"2024-03-29T05:00:43.914064+0000","last_became_peered":"2024-03-29T05:00:43.914064+0000","last_unstale":"2024-03-29T05:10:22.730729+0000","last_undegraded":"2024-03-29T05:10:22.730729+0000","last_fullsized":"2024-03-29T05:10:22.730729+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:39:51.614460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1330,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.864626+0000","last_change":"2024-03-29T05:01:53.840294+0000","last_active":"2024-03-29T05:10:26.864626+0000","last_peered":"2024-03-29T05:10:26.864626+0000","last_clean":"2024-03-29T05:10:26.864626+0000","last_became_active":"2024-03-29T05:01:53.840016+0000","last_became_peered":"2024-03-29T05:01:53.840016+0000","last_unstale":"2024-03-29T05:10:26.864626+0000","last_undegraded":"2024-03-29T05:10:26.864626+0000","last_fullsized":"2024-03-29T05:10:26.864626+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:06:30.265575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1446,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.802461+0000","last_change":"2024-03-29T05:01:53.822039+0000","last_active":"2024-03-29T05:10:26.802461+0000","last_peered":"2024-03-29T05:10:26.802461+0000","last_clean":"2024-03-29T05:10:26.802461+0000","last_became_active":"2024-03-29T05:01:53.821657+0000","last_became_peered":"2024-03-29T05:01:53.821657+0000","last_unstale":"2024-03-29T05:10:26.802461+0000","last_undegraded":"2024-03-29T05:10:26.802461+0000","last_fullsized":"2024-03-29T05:10:26.802461+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:00:15.680313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":922,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.804997+0000","last_change":"2024-03-29T05:00:43.916081+0000","last_active":"2024-03-29T05:10:22.804997+0000","last_peered":"2024-03-29T05:10:22.804997+0000","last_clean":"2024-03-29T05:10:22.804997+0000","last_became_active":"2024-03-29T05:00:43.915222+0000","last_became_peered":"2024-03-29T05:00:43.915222+0000","last_unstale":"2024-03-29T05:10:22.804997+0000","last_undegraded":"2024-03-29T05:10:22.804997+0000","last_fullsized":"2024-03-29T05:10:22.804997+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:23:17.449789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1348,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.864627+0000","last_change":"2024-03-29T05:01:53.826966+0000","last_active":"2024-03-29T05:10:26.864627+0000","last_peered":"2024-03-29T05:10:26.864627+0000","last_clean":"2024-03-29T05:10:26.864627+0000","last_became_active":"2024-03-29T05:01:53.826460+0000","last_became_peered":"2024-03-29T05:01:53.826460+0000","last_unstale":"2024-03-29T05:10:26.864627+0000","last_undegraded":"2024-03-29T05:10:26.864627+0000","last_fullsized":"2024-03-29T05:10:26.864627+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:36:08.169321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1448,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.801804+0000","last_change":"2024-03-29T05:01:53.828459+0000","last_active":"2024-03-29T05:10:26.801804+0000","last_peered":"2024-03-29T05:10:26.801804+0000","last_clean":"2024-03-29T05:10:26.801804+0000","last_became_active":"2024-03-29T05:01:53.828149+0000","last_became_peered":"2024-03-29T05:01:53.828149+0000","last_unstale":"2024-03-29T05:10:26.801804+0000","last_undegraded":"2024-03-29T05:10:26.801804+0000","last_fullsized":"2024-03-29T05:10:26.801804+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:49:51.889159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1407,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.084617+0000","last_change":"2024-03-29T05:02:15.249825+0000","last_active":"2024-03-29T05:10:26.084617+0000","last_peered":"2024-03-29T05:10:26.084617+0000","last_clean":"2024-03-29T05:10:26.084617+0000","last_became_active":"2024-03-29T05:02:15.249461+0000","last_became_peered":"2024-03-29T05:02:15.249461+0000","last_unstale":"2024-03-29T05:10:26.084617+0000","last_undegraded":"2024-03-29T05:10:26.084617+0000","last_fullsized":"2024-03-29T05:10:26.084617+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:24:19.894615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1342,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.865151+0000","last_change":"2024-03-29T05:02:15.258674+0000","last_active":"2024-03-29T05:10:26.865151+0000","last_peered":"2024-03-29T05:10:26.865151+0000","last_clean":"2024-03-29T05:10:26.865151+0000","last_became_active":"2024-03-29T05:02:15.257963+0000","last_became_peered":"2024-03-29T05:02:15.257963+0000","last_unstale":"2024-03-29T05:10:26.865151+0000","last_undegraded":"2024-03-29T05:10:26.865151+0000","last_fullsized":"2024-03-29T05:10:26.865151+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:29:43.805122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":1310,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.045027+0000","last_change":"2024-03-29T05:02:15.250743+0000","last_active":"2024-03-29T05:10:26.045027+0000","last_peered":"2024-03-29T05:10:26.045027+0000","last_clean":"2024-03-29T05:10:26.045027+0000","last_became_active":"2024-03-29T05:02:15.250374+0000","last_became_peered":"2024-03-29T05:02:15.250374+0000","last_unstale":"2024-03-29T05:10:26.045027+0000","last_undegraded":"2024-03-29T05:10:26.045027+0000","last_fullsized":"2024-03-29T05:10:26.045027+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:02:27.429005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2052,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201536,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":322256,"kb_used_data":8516,"kb_used_omap":1,"kb_used_meta":313726,"kb_avail":93422896,"statfs":{"total":95995035648,"available":95665045504,"internally_reserved":0,"allocated":8720384,"data_stored":6866451,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":321255768},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463174,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":318144,"kb_used_data":8504,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93427008,"statfs":{"total":95995035648,"available":95669256192,"internally_reserved":0,"allocated":8708096,"data_stored":6866305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":490,"seq":2104533975167,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":59392,"kb_used_data":9456,"kb_used_omap":53,"kb_used_meta":49866,"kb_avail":93685760,"statfs":{"total":95995035648,"available":95934218240,"internally_reserved":0,"allocated":9682944,"data_stored":6500300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55167,"internal_metadata":51062913},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":425,"seq":1825361100941,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":55712,"kb_used_data":10068,"kb_used_omap":42,"kb_used_meta":45589,"kb_avail":93689440,"statfs":{"total":95995035648,"available":95937986560,"internally_reserved":0,"allocated":10309632,"data_stored":7096084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43631,"internal_metadata":46683537},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":525,"seq":2254857830520,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":63784,"kb_used_data":9628,"kb_used_omap":64,"kb_used_meta":54079,"kb_avail":93681368,"statfs":{"total":95995035648,"available":95929720832,"internally_reserved":0,"allocated":9859072,"data_stored":6526557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":66188,"internal_metadata":55377268},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":555,"seq":2383706849393,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":59232,"kb_used_data":9364,"kb_used_omap":55,"kb_used_meta":49800,"kb_avail":93685920,"statfs":{"total":95995035648,"available":95934382080,"internally_reserved":0,"allocated":9588736,"data_stored":6489166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56870,"internal_metadata":50995674},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":576,"seq":2473901162605,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":59364,"kb_used_data":9560,"kb_used_omap":56,"kb_used_meta":49735,"kb_avail":93685788,"statfs":{"total":95995035648,"available":95934246912,"internally_reserved":0,"allocated":9789440,"data_stored":6515747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58303,"internal_metadata":50928705},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724810,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":317560,"kb_used_data":7916,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93427592,"statfs":{"total":95995035648,"available":95669854208,"internally_reserved":0,"allocated":8105984,"data_stored":6271763,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":706,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1346,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T05:11:12.302 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T05:11:12.559 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:11:12.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:12 smithi181 ceph-mon[101970]: from='client.44783 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T05:11:12.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:12 smithi181 ceph-mon[106466]: from='client.44783 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T05:11:13.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:12 smithi187 ceph-mon[93536]: from='client.44783 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T05:11:13.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:13 smithi181 ceph-mon[101970]: pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:13.702 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:13 smithi181 ceph-mon[106466]: pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:14.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:13 smithi187 ceph-mon[93536]: pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:14.391 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:11:14.392 INFO:teuthology.orchestra.run.smithi181.stderr:dumped all 2024-03-29T05:11:15.107 INFO:teuthology.orchestra.run.smithi181.stdout:{"pg_ready":true,"pg_map":{"version":1215,"stamp":"2024-03-29T05:11:12.551873+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":300,"num_write_kb":2445,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":216,"ondisk_log_size":216,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1255444,"kb_used_data":73012,"kb_used_omap":278,"kb_used_meta":1182057,"kb_avail":748705772,"statfs":{"total":767960285184,"available":766674710528,"internally_reserved":0,"allocated":74764288,"data_stored":53132373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":285271,"internal_metadata":1210426793},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002414"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":1298,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.045549+0000","last_change":"2024-03-29T05:02:15.260835+0000","last_active":"2024-03-29T05:10:26.045549+0000","last_peered":"2024-03-29T05:10:26.045549+0000","last_clean":"2024-03-29T05:10:26.045549+0000","last_became_active":"2024-03-29T05:02:15.260504+0000","last_became_peered":"2024-03-29T05:02:15.260504+0000","last_unstale":"2024-03-29T05:10:26.045549+0000","last_undegraded":"2024-03-29T05:10:26.045549+0000","last_fullsized":"2024-03-29T05:10:26.045549+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:49:46.828819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":1454,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.802346+0000","last_change":"2024-03-29T05:01:18.501548+0000","last_active":"2024-03-29T05:10:26.802346+0000","last_peered":"2024-03-29T05:10:26.802346+0000","last_clean":"2024-03-29T05:10:26.802346+0000","last_became_active":"2024-03-29T05:01:18.501159+0000","last_became_peered":"2024-03-29T05:01:18.501159+0000","last_unstale":"2024-03-29T05:10:26.802346+0000","last_undegraded":"2024-03-29T05:10:26.802346+0000","last_fullsized":"2024-03-29T05:10:26.802346+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:22:10.843117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1340,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.865715+0000","last_change":"2024-03-29T05:01:53.840366+0000","last_active":"2024-03-29T05:10:26.865715+0000","last_peered":"2024-03-29T05:10:26.865715+0000","last_clean":"2024-03-29T05:10:26.865715+0000","last_became_active":"2024-03-29T05:01:53.840030+0000","last_became_peered":"2024-03-29T05:01:53.840030+0000","last_unstale":"2024-03-29T05:10:26.865715+0000","last_undegraded":"2024-03-29T05:10:26.865715+0000","last_fullsized":"2024-03-29T05:10:26.865715+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:15:15.057546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":1326,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.865573+0000","last_change":"2024-03-29T05:01:53.839814+0000","last_active":"2024-03-29T05:10:26.865573+0000","last_peered":"2024-03-29T05:10:26.865573+0000","last_clean":"2024-03-29T05:10:26.865573+0000","last_became_active":"2024-03-29T05:01:53.839486+0000","last_became_peered":"2024-03-29T05:01:53.839486+0000","last_unstale":"2024-03-29T05:10:26.865573+0000","last_undegraded":"2024-03-29T05:10:26.865573+0000","last_fullsized":"2024-03-29T05:10:26.865573+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:53:01.662020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1338,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.045570+0000","last_change":"2024-03-29T05:02:15.261932+0000","last_active":"2024-03-29T05:10:26.045570+0000","last_peered":"2024-03-29T05:10:26.045570+0000","last_clean":"2024-03-29T05:10:26.045570+0000","last_became_active":"2024-03-29T05:02:15.261350+0000","last_became_peered":"2024-03-29T05:02:15.261350+0000","last_unstale":"2024-03-29T05:10:26.045570+0000","last_undegraded":"2024-03-29T05:10:26.045570+0000","last_fullsized":"2024-03-29T05:10:26.045570+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:56:31.965997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":928,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.731173+0000","last_change":"2024-03-29T05:02:15.258277+0000","last_active":"2024-03-29T05:10:22.731173+0000","last_peered":"2024-03-29T05:10:22.731173+0000","last_clean":"2024-03-29T05:10:22.731173+0000","last_became_active":"2024-03-29T05:02:15.257769+0000","last_became_peered":"2024-03-29T05:02:15.257769+0000","last_unstale":"2024-03-29T05:10:22.731173+0000","last_undegraded":"2024-03-29T05:10:22.731173+0000","last_fullsized":"2024-03-29T05:10:22.731173+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:37:01.504077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1338,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.044709+0000","last_change":"2024-03-29T05:02:15.264103+0000","last_active":"2024-03-29T05:10:26.044709+0000","last_peered":"2024-03-29T05:10:26.044709+0000","last_clean":"2024-03-29T05:10:26.044709+0000","last_became_active":"2024-03-29T05:02:15.258201+0000","last_became_peered":"2024-03-29T05:02:15.258201+0000","last_unstale":"2024-03-29T05:10:26.044709+0000","last_undegraded":"2024-03-29T05:10:26.044709+0000","last_fullsized":"2024-03-29T05:10:26.044709+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:03:15.410934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":945,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.730778+0000","last_change":"2024-03-29T05:02:15.262078+0000","last_active":"2024-03-29T05:10:22.730778+0000","last_peered":"2024-03-29T05:10:22.730778+0000","last_clean":"2024-03-29T05:10:22.730778+0000","last_became_active":"2024-03-29T05:02:15.261728+0000","last_became_peered":"2024-03-29T05:02:15.261728+0000","last_unstale":"2024-03-29T05:10:22.730778+0000","last_undegraded":"2024-03-29T05:10:22.730778+0000","last_fullsized":"2024-03-29T05:10:22.730778+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:20:28.781180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1362,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.038934+0000","last_change":"2024-03-29T05:01:53.832409+0000","last_active":"2024-03-29T05:10:26.038934+0000","last_peered":"2024-03-29T05:10:26.038934+0000","last_clean":"2024-03-29T05:10:26.038934+0000","last_became_active":"2024-03-29T05:01:53.832075+0000","last_became_peered":"2024-03-29T05:01:53.832075+0000","last_unstale":"2024-03-29T05:10:26.038934+0000","last_undegraded":"2024-03-29T05:10:26.038934+0000","last_fullsized":"2024-03-29T05:10:26.038934+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:27:38.179605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":1450,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.801642+0000","last_change":"2024-03-29T05:02:15.263522+0000","last_active":"2024-03-29T05:10:26.801642+0000","last_peered":"2024-03-29T05:10:26.801642+0000","last_clean":"2024-03-29T05:10:26.801642+0000","last_became_active":"2024-03-29T05:02:15.263094+0000","last_became_peered":"2024-03-29T05:02:15.263094+0000","last_unstale":"2024-03-29T05:10:26.801642+0000","last_undegraded":"2024-03-29T05:10:26.801642+0000","last_fullsized":"2024-03-29T05:10:26.801642+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:02:19.808048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1309,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.044263+0000","last_change":"2024-03-29T05:02:15.258718+0000","last_active":"2024-03-29T05:10:26.044263+0000","last_peered":"2024-03-29T05:10:26.044263+0000","last_clean":"2024-03-29T05:10:26.044263+0000","last_became_active":"2024-03-29T05:02:15.256411+0000","last_became_peered":"2024-03-29T05:02:15.256411+0000","last_unstale":"2024-03-29T05:10:26.044263+0000","last_undegraded":"2024-03-29T05:10:26.044263+0000","last_fullsized":"2024-03-29T05:10:26.044263+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:08:52.734498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":1460,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.801223+0000","last_change":"2024-03-29T05:02:15.259190+0000","last_active":"2024-03-29T05:10:26.801223+0000","last_peered":"2024-03-29T05:10:26.801223+0000","last_clean":"2024-03-29T05:10:26.801223+0000","last_became_active":"2024-03-29T05:02:15.258814+0000","last_became_peered":"2024-03-29T05:02:15.258814+0000","last_unstale":"2024-03-29T05:10:26.801223+0000","last_undegraded":"2024-03-29T05:10:26.801223+0000","last_fullsized":"2024-03-29T05:10:26.801223+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:21:54.355754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":927,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.805092+0000","last_change":"2024-03-29T05:01:18.509131+0000","last_active":"2024-03-29T05:10:22.805092+0000","last_peered":"2024-03-29T05:10:22.805092+0000","last_clean":"2024-03-29T05:10:22.805092+0000","last_became_active":"2024-03-29T05:01:18.508777+0000","last_became_peered":"2024-03-29T05:01:18.508777+0000","last_unstale":"2024-03-29T05:10:22.805092+0000","last_undegraded":"2024-03-29T05:10:22.805092+0000","last_fullsized":"2024-03-29T05:10:22.805092+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:33:48.034434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1307,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.864748+0000","last_change":"2024-03-29T05:01:53.838294+0000","last_active":"2024-03-29T05:10:26.864748+0000","last_peered":"2024-03-29T05:10:26.864748+0000","last_clean":"2024-03-29T05:10:26.864748+0000","last_became_active":"2024-03-29T05:01:53.834516+0000","last_became_peered":"2024-03-29T05:01:53.834516+0000","last_unstale":"2024-03-29T05:10:26.864748+0000","last_undegraded":"2024-03-29T05:10:26.864748+0000","last_fullsized":"2024-03-29T05:10:26.864748+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:02:39.369882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1398,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.085442+0000","last_change":"2024-03-29T05:02:15.252323+0000","last_active":"2024-03-29T05:10:26.085442+0000","last_peered":"2024-03-29T05:10:26.085442+0000","last_clean":"2024-03-29T05:10:26.085442+0000","last_became_active":"2024-03-29T05:02:15.251982+0000","last_became_peered":"2024-03-29T05:02:15.251982+0000","last_unstale":"2024-03-29T05:10:26.085442+0000","last_undegraded":"2024-03-29T05:10:26.085442+0000","last_fullsized":"2024-03-29T05:10:26.085442+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:54:20.450633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":1310,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.865753+0000","last_change":"2024-03-29T05:01:53.836394+0000","last_active":"2024-03-29T05:10:26.865753+0000","last_peered":"2024-03-29T05:10:26.865753+0000","last_clean":"2024-03-29T05:10:26.865753+0000","last_became_active":"2024-03-29T05:01:53.834444+0000","last_became_peered":"2024-03-29T05:01:53.834444+0000","last_unstale":"2024-03-29T05:10:26.865753+0000","last_undegraded":"2024-03-29T05:10:26.865753+0000","last_fullsized":"2024-03-29T05:10:26.865753+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:02:57.479683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1439,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.802147+0000","last_change":"2024-03-29T04:59:33.396754+0000","last_active":"2024-03-29T05:10:26.802147+0000","last_peered":"2024-03-29T05:10:26.802147+0000","last_clean":"2024-03-29T05:10:26.802147+0000","last_became_active":"2024-03-29T04:59:33.395620+0000","last_became_peered":"2024-03-29T04:59:33.395620+0000","last_unstale":"2024-03-29T05:10:26.802147+0000","last_undegraded":"2024-03-29T05:10:26.802147+0000","last_fullsized":"2024-03-29T05:10:26.802147+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:32:54.303599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1399,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.085001+0000","last_change":"2024-03-29T05:01:18.501816+0000","last_active":"2024-03-29T05:10:26.085001+0000","last_peered":"2024-03-29T05:10:26.085001+0000","last_clean":"2024-03-29T05:10:26.085001+0000","last_became_active":"2024-03-29T05:01:18.501353+0000","last_became_peered":"2024-03-29T05:01:18.501353+0000","last_unstale":"2024-03-29T05:10:26.085001+0000","last_undegraded":"2024-03-29T05:10:26.085001+0000","last_fullsized":"2024-03-29T05:10:26.085001+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:38:26.980925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":941,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.090830+0000","last_change":"2024-03-29T05:02:15.261144+0000","last_active":"2024-03-29T05:10:23.090830+0000","last_peered":"2024-03-29T05:10:23.090830+0000","last_clean":"2024-03-29T05:10:23.090830+0000","last_became_active":"2024-03-29T05:02:15.260781+0000","last_became_peered":"2024-03-29T05:02:15.260781+0000","last_unstale":"2024-03-29T05:10:23.090830+0000","last_undegraded":"2024-03-29T05:10:23.090830+0000","last_fullsized":"2024-03-29T05:10:23.090830+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:25:19.644980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1461,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.801918+0000","last_change":"2024-03-29T05:01:18.505730+0000","last_active":"2024-03-29T05:10:26.801918+0000","last_peered":"2024-03-29T05:10:26.801918+0000","last_clean":"2024-03-29T05:10:26.801918+0000","last_became_active":"2024-03-29T05:01:18.505403+0000","last_became_peered":"2024-03-29T05:01:18.505403+0000","last_unstale":"2024-03-29T05:10:26.801918+0000","last_undegraded":"2024-03-29T05:10:26.801918+0000","last_fullsized":"2024-03-29T05:10:26.801918+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:15:52.154347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1456,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.802479+0000","last_change":"2024-03-29T05:02:15.258159+0000","last_active":"2024-03-29T05:10:26.802479+0000","last_peered":"2024-03-29T05:10:26.802479+0000","last_clean":"2024-03-29T05:10:26.802479+0000","last_became_active":"2024-03-29T05:02:15.257780+0000","last_became_peered":"2024-03-29T05:02:15.257780+0000","last_unstale":"2024-03-29T05:10:26.802479+0000","last_undegraded":"2024-03-29T05:10:26.802479+0000","last_fullsized":"2024-03-29T05:10:26.802479+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:32:25.749061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":941,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.090022+0000","last_change":"2024-03-29T05:00:43.919799+0000","last_active":"2024-03-29T05:10:23.090022+0000","last_peered":"2024-03-29T05:10:23.090022+0000","last_clean":"2024-03-29T05:10:23.090022+0000","last_became_active":"2024-03-29T05:00:43.919485+0000","last_became_peered":"2024-03-29T05:00:43.919485+0000","last_unstale":"2024-03-29T05:10:23.090022+0000","last_undegraded":"2024-03-29T05:10:23.090022+0000","last_fullsized":"2024-03-29T05:10:23.090022+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:26:33.257813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":922,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.805039+0000","last_change":"2024-03-29T05:02:15.266302+0000","last_active":"2024-03-29T05:10:22.805039+0000","last_peered":"2024-03-29T05:10:22.805039+0000","last_clean":"2024-03-29T05:10:22.805039+0000","last_became_active":"2024-03-29T05:02:15.266015+0000","last_became_peered":"2024-03-29T05:02:15.266015+0000","last_unstale":"2024-03-29T05:10:22.805039+0000","last_undegraded":"2024-03-29T05:10:22.805039+0000","last_fullsized":"2024-03-29T05:10:22.805039+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:46:54.101383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1300,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.864719+0000","last_change":"2024-03-29T05:01:53.840657+0000","last_active":"2024-03-29T05:10:26.864719+0000","last_peered":"2024-03-29T05:10:26.864719+0000","last_clean":"2024-03-29T05:10:26.864719+0000","last_became_active":"2024-03-29T05:01:53.838284+0000","last_became_peered":"2024-03-29T05:01:53.838284+0000","last_unstale":"2024-03-29T05:10:26.864719+0000","last_undegraded":"2024-03-29T05:10:26.864719+0000","last_fullsized":"2024-03-29T05:10:26.864719+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:16:16.577458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"247'161","reported_seq":1301,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.090050+0000","last_change":"2024-03-29T04:59:33.398429+0000","last_active":"2024-03-29T05:10:23.090050+0000","last_peered":"2024-03-29T05:10:23.090050+0000","last_clean":"2024-03-29T05:10:23.090050+0000","last_became_active":"2024-03-29T04:59:33.397970+0000","last_became_peered":"2024-03-29T04:59:33.397970+0000","last_unstale":"2024-03-29T05:10:23.090050+0000","last_undegraded":"2024-03-29T05:10:23.090050+0000","last_fullsized":"2024-03-29T05:10:23.090050+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:48:01.601528+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:48:01.601528+0000","last_clean_scrub_stamp":"2024-03-29T04:48:01.601528+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:42:56.518950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":930,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.730065+0000","last_change":"2024-03-29T05:01:18.506639+0000","last_active":"2024-03-29T05:10:22.730065+0000","last_peered":"2024-03-29T05:10:22.730065+0000","last_clean":"2024-03-29T05:10:22.730065+0000","last_became_active":"2024-03-29T05:01:18.506276+0000","last_became_peered":"2024-03-29T05:01:18.506276+0000","last_unstale":"2024-03-29T05:10:22.730065+0000","last_undegraded":"2024-03-29T05:10:22.730065+0000","last_fullsized":"2024-03-29T05:10:22.730065+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:10:41.303709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1396,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.086255+0000","last_change":"2024-03-29T05:00:43.908190+0000","last_active":"2024-03-29T05:10:26.086255+0000","last_peered":"2024-03-29T05:10:26.086255+0000","last_clean":"2024-03-29T05:10:26.086255+0000","last_became_active":"2024-03-29T05:00:43.907883+0000","last_became_peered":"2024-03-29T05:00:43.907883+0000","last_unstale":"2024-03-29T05:10:26.086255+0000","last_undegraded":"2024-03-29T05:10:26.086255+0000","last_fullsized":"2024-03-29T05:10:26.086255+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:31:27.009072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1399,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.084741+0000","last_change":"2024-03-29T05:01:18.509368+0000","last_active":"2024-03-29T05:10:26.084741+0000","last_peered":"2024-03-29T05:10:26.084741+0000","last_clean":"2024-03-29T05:10:26.084741+0000","last_became_active":"2024-03-29T05:01:18.508981+0000","last_became_peered":"2024-03-29T05:01:18.508981+0000","last_unstale":"2024-03-29T05:10:26.084741+0000","last_undegraded":"2024-03-29T05:10:26.084741+0000","last_fullsized":"2024-03-29T05:10:26.084741+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:33:33.633755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1386,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.038752+0000","last_change":"2024-03-29T05:01:53.832886+0000","last_active":"2024-03-29T05:10:26.038752+0000","last_peered":"2024-03-29T05:10:26.038752+0000","last_clean":"2024-03-29T05:10:26.038752+0000","last_became_active":"2024-03-29T05:01:53.832568+0000","last_became_peered":"2024-03-29T05:01:53.832568+0000","last_unstale":"2024-03-29T05:10:26.038752+0000","last_undegraded":"2024-03-29T05:10:26.038752+0000","last_fullsized":"2024-03-29T05:10:26.038752+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:20:06.656000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":939,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.730470+0000","last_change":"2024-03-29T05:01:53.824832+0000","last_active":"2024-03-29T05:10:22.730470+0000","last_peered":"2024-03-29T05:10:22.730470+0000","last_clean":"2024-03-29T05:10:22.730470+0000","last_became_active":"2024-03-29T05:01:53.824486+0000","last_became_peered":"2024-03-29T05:01:53.824486+0000","last_unstale":"2024-03-29T05:10:22.730470+0000","last_undegraded":"2024-03-29T05:10:22.730470+0000","last_fullsized":"2024-03-29T05:10:22.730470+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:23:48.429185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1317,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.045660+0000","last_change":"2024-03-29T05:02:15.264450+0000","last_active":"2024-03-29T05:10:26.045660+0000","last_peered":"2024-03-29T05:10:26.045660+0000","last_clean":"2024-03-29T05:10:26.045660+0000","last_became_active":"2024-03-29T05:02:15.261877+0000","last_became_peered":"2024-03-29T05:02:15.261877+0000","last_unstale":"2024-03-29T05:10:26.045660+0000","last_undegraded":"2024-03-29T05:10:26.045660+0000","last_fullsized":"2024-03-29T05:10:26.045660+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:55:23.320598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":930,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.730065+0000","last_change":"2024-03-29T05:00:43.922404+0000","last_active":"2024-03-29T05:10:22.730065+0000","last_peered":"2024-03-29T05:10:22.730065+0000","last_clean":"2024-03-29T05:10:22.730065+0000","last_became_active":"2024-03-29T05:00:43.922095+0000","last_became_peered":"2024-03-29T05:00:43.922095+0000","last_unstale":"2024-03-29T05:10:22.730065+0000","last_undegraded":"2024-03-29T05:10:22.730065+0000","last_fullsized":"2024-03-29T05:10:22.730065+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:34:54.348996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1399,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.085022+0000","last_change":"2024-03-29T05:02:15.249516+0000","last_active":"2024-03-29T05:10:26.085022+0000","last_peered":"2024-03-29T05:10:26.085022+0000","last_clean":"2024-03-29T05:10:26.085022+0000","last_became_active":"2024-03-29T05:02:15.249143+0000","last_became_peered":"2024-03-29T05:02:15.249143+0000","last_unstale":"2024-03-29T05:10:26.085022+0000","last_undegraded":"2024-03-29T05:10:26.085022+0000","last_fullsized":"2024-03-29T05:10:26.085022+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:26:36.986495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":943,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.731091+0000","last_change":"2024-03-29T05:01:53.826633+0000","last_active":"2024-03-29T05:10:22.731091+0000","last_peered":"2024-03-29T05:10:22.731091+0000","last_clean":"2024-03-29T05:10:22.731091+0000","last_became_active":"2024-03-29T05:01:53.826358+0000","last_became_peered":"2024-03-29T05:01:53.826358+0000","last_unstale":"2024-03-29T05:10:22.731091+0000","last_undegraded":"2024-03-29T05:10:22.731091+0000","last_fullsized":"2024-03-29T05:10:22.731091+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:04:56.270939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1402,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.085362+0000","last_change":"2024-03-29T05:00:43.910959+0000","last_active":"2024-03-29T05:10:26.085362+0000","last_peered":"2024-03-29T05:10:26.085362+0000","last_clean":"2024-03-29T05:10:26.085362+0000","last_became_active":"2024-03-29T05:00:43.909800+0000","last_became_peered":"2024-03-29T05:00:43.909800+0000","last_unstale":"2024-03-29T05:10:26.085362+0000","last_undegraded":"2024-03-29T05:10:26.085362+0000","last_fullsized":"2024-03-29T05:10:26.085362+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:50:38.028650+0000","last_clean_scrub_stamp":"2024-03-29T04:50:38.028650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:37:21.817383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1407,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.086161+0000","last_change":"2024-03-29T05:02:15.249702+0000","last_active":"2024-03-29T05:10:26.086161+0000","last_peered":"2024-03-29T05:10:26.086161+0000","last_clean":"2024-03-29T05:10:26.086161+0000","last_became_active":"2024-03-29T05:02:15.249389+0000","last_became_peered":"2024-03-29T05:02:15.249389+0000","last_unstale":"2024-03-29T05:10:26.086161+0000","last_undegraded":"2024-03-29T05:10:26.086161+0000","last_fullsized":"2024-03-29T05:10:26.086161+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:21:44.015645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1332,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.864795+0000","last_change":"2024-03-29T05:01:53.839868+0000","last_active":"2024-03-29T05:10:26.864795+0000","last_peered":"2024-03-29T05:10:26.864795+0000","last_clean":"2024-03-29T05:10:26.864795+0000","last_became_active":"2024-03-29T05:01:53.837259+0000","last_became_peered":"2024-03-29T05:01:53.837259+0000","last_unstale":"2024-03-29T05:10:26.864795+0000","last_undegraded":"2024-03-29T05:10:26.864795+0000","last_fullsized":"2024-03-29T05:10:26.864795+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:49:29.224580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1298,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.045704+0000","last_change":"2024-03-29T05:02:15.258417+0000","last_active":"2024-03-29T05:10:26.045704+0000","last_peered":"2024-03-29T05:10:26.045704+0000","last_clean":"2024-03-29T05:10:26.045704+0000","last_became_active":"2024-03-29T05:02:15.257620+0000","last_became_peered":"2024-03-29T05:02:15.257620+0000","last_unstale":"2024-03-29T05:10:26.045704+0000","last_undegraded":"2024-03-29T05:10:26.045704+0000","last_fullsized":"2024-03-29T05:10:26.045704+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:58:46.529609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1293,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.044347+0000","last_change":"2024-03-29T05:02:15.256760+0000","last_active":"2024-03-29T05:10:26.044347+0000","last_peered":"2024-03-29T05:10:26.044347+0000","last_clean":"2024-03-29T05:10:26.044347+0000","last_became_active":"2024-03-29T05:02:15.255198+0000","last_became_peered":"2024-03-29T05:02:15.255198+0000","last_unstale":"2024-03-29T05:10:26.044347+0000","last_undegraded":"2024-03-29T05:10:26.044347+0000","last_fullsized":"2024-03-29T05:10:26.044347+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:26:40.696773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1308,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.044433+0000","last_change":"2024-03-29T05:02:15.256138+0000","last_active":"2024-03-29T05:10:26.044433+0000","last_peered":"2024-03-29T05:10:26.044433+0000","last_clean":"2024-03-29T05:10:26.044433+0000","last_became_active":"2024-03-29T05:02:15.254255+0000","last_became_peered":"2024-03-29T05:02:15.254255+0000","last_unstale":"2024-03-29T05:10:26.044433+0000","last_undegraded":"2024-03-29T05:10:26.044433+0000","last_fullsized":"2024-03-29T05:10:26.044433+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:45:49.901834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":940,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.805347+0000","last_change":"2024-03-29T05:02:15.268430+0000","last_active":"2024-03-29T05:10:22.805347+0000","last_peered":"2024-03-29T05:10:22.805347+0000","last_clean":"2024-03-29T05:10:22.805347+0000","last_became_active":"2024-03-29T05:02:15.268113+0000","last_became_peered":"2024-03-29T05:02:15.268113+0000","last_unstale":"2024-03-29T05:10:22.805347+0000","last_undegraded":"2024-03-29T05:10:22.805347+0000","last_fullsized":"2024-03-29T05:10:22.805347+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:48:31.192534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1320,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.044611+0000","last_change":"2024-03-29T05:02:15.261131+0000","last_active":"2024-03-29T05:10:26.044611+0000","last_peered":"2024-03-29T05:10:26.044611+0000","last_clean":"2024-03-29T05:10:26.044611+0000","last_became_active":"2024-03-29T05:02:15.260379+0000","last_became_peered":"2024-03-29T05:02:15.260379+0000","last_unstale":"2024-03-29T05:10:26.044611+0000","last_undegraded":"2024-03-29T05:10:26.044611+0000","last_fullsized":"2024-03-29T05:10:26.044611+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:27:11.830847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":1319,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.866207+0000","last_change":"2024-03-29T05:02:15.266435+0000","last_active":"2024-03-29T05:10:26.866207+0000","last_peered":"2024-03-29T05:10:26.866207+0000","last_clean":"2024-03-29T05:10:26.866207+0000","last_became_active":"2024-03-29T05:02:15.266112+0000","last_became_peered":"2024-03-29T05:02:15.266112+0000","last_unstale":"2024-03-29T05:10:26.866207+0000","last_undegraded":"2024-03-29T05:10:26.866207+0000","last_fullsized":"2024-03-29T05:10:26.866207+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:09:33.391187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1457,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.801846+0000","last_change":"2024-03-29T05:02:15.265027+0000","last_active":"2024-03-29T05:10:26.801846+0000","last_peered":"2024-03-29T05:10:26.801846+0000","last_clean":"2024-03-29T05:10:26.801846+0000","last_became_active":"2024-03-29T05:02:15.264664+0000","last_became_peered":"2024-03-29T05:02:15.264664+0000","last_unstale":"2024-03-29T05:10:26.801846+0000","last_undegraded":"2024-03-29T05:10:26.801846+0000","last_fullsized":"2024-03-29T05:10:26.801846+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:54:41.748257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":926,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.730643+0000","last_change":"2024-03-29T05:01:53.824320+0000","last_active":"2024-03-29T05:10:22.730643+0000","last_peered":"2024-03-29T05:10:22.730643+0000","last_clean":"2024-03-29T05:10:22.730643+0000","last_became_active":"2024-03-29T05:01:53.824010+0000","last_became_peered":"2024-03-29T05:01:53.824010+0000","last_unstale":"2024-03-29T05:10:22.730643+0000","last_undegraded":"2024-03-29T05:10:22.730643+0000","last_fullsized":"2024-03-29T05:10:22.730643+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:20:59.972126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":922,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.805214+0000","last_change":"2024-03-29T05:00:43.916507+0000","last_active":"2024-03-29T05:10:22.805214+0000","last_peered":"2024-03-29T05:10:22.805214+0000","last_clean":"2024-03-29T05:10:22.805214+0000","last_became_active":"2024-03-29T05:00:43.916027+0000","last_became_peered":"2024-03-29T05:00:43.916027+0000","last_unstale":"2024-03-29T05:10:22.805214+0000","last_undegraded":"2024-03-29T05:10:22.805214+0000","last_fullsized":"2024-03-29T05:10:22.805214+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:12:04.043477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":904,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.089792+0000","last_change":"2024-03-29T04:51:50.896476+0000","last_active":"2024-03-29T05:10:23.089792+0000","last_peered":"2024-03-29T05:10:23.089792+0000","last_clean":"2024-03-29T05:10:23.089792+0000","last_became_active":"2024-03-29T04:51:50.895258+0000","last_became_peered":"2024-03-29T04:51:50.895258+0000","last_unstale":"2024-03-29T05:10:23.089792+0000","last_undegraded":"2024-03-29T05:10:23.089792+0000","last_fullsized":"2024-03-29T05:10:23.089792+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:24:42.583134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1339,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.044132+0000","last_change":"2024-03-29T05:02:15.257042+0000","last_active":"2024-03-29T05:10:26.044132+0000","last_peered":"2024-03-29T05:10:26.044132+0000","last_clean":"2024-03-29T05:10:26.044132+0000","last_became_active":"2024-03-29T05:02:15.255472+0000","last_became_peered":"2024-03-29T05:02:15.255472+0000","last_unstale":"2024-03-29T05:10:26.044132+0000","last_undegraded":"2024-03-29T05:10:26.044132+0000","last_fullsized":"2024-03-29T05:10:26.044132+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:34:22.131400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":939,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.090843+0000","last_change":"2024-03-29T05:02:15.260429+0000","last_active":"2024-03-29T05:10:23.090843+0000","last_peered":"2024-03-29T05:10:23.090843+0000","last_clean":"2024-03-29T05:10:23.090843+0000","last_became_active":"2024-03-29T05:02:15.259772+0000","last_became_peered":"2024-03-29T05:02:15.259772+0000","last_unstale":"2024-03-29T05:10:23.090843+0000","last_undegraded":"2024-03-29T05:10:23.090843+0000","last_fullsized":"2024-03-29T05:10:23.090843+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:16:01.290463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":908,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.730407+0000","last_change":"2024-03-29T04:51:51.764872+0000","last_active":"2024-03-29T05:10:22.730407+0000","last_peered":"2024-03-29T05:10:22.730407+0000","last_clean":"2024-03-29T05:10:22.730407+0000","last_became_active":"2024-03-29T04:51:51.764554+0000","last_became_peered":"2024-03-29T04:51:51.764554+0000","last_unstale":"2024-03-29T05:10:22.730407+0000","last_undegraded":"2024-03-29T05:10:22.730407+0000","last_fullsized":"2024-03-29T05:10:22.730407+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:09:07.383326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":938,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.090913+0000","last_change":"2024-03-29T05:02:15.260665+0000","last_active":"2024-03-29T05:10:23.090913+0000","last_peered":"2024-03-29T05:10:23.090913+0000","last_clean":"2024-03-29T05:10:23.090913+0000","last_became_active":"2024-03-29T05:02:15.260149+0000","last_became_peered":"2024-03-29T05:02:15.260149+0000","last_unstale":"2024-03-29T05:10:23.090913+0000","last_undegraded":"2024-03-29T05:10:23.090913+0000","last_fullsized":"2024-03-29T05:10:23.090913+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:39:03.132413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1330,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.865624+0000","last_change":"2024-03-29T05:01:53.830781+0000","last_active":"2024-03-29T05:10:26.865624+0000","last_peered":"2024-03-29T05:10:26.865624+0000","last_clean":"2024-03-29T05:10:26.865624+0000","last_became_active":"2024-03-29T05:01:53.830556+0000","last_became_peered":"2024-03-29T05:01:53.830556+0000","last_unstale":"2024-03-29T05:10:26.865624+0000","last_undegraded":"2024-03-29T05:10:26.865624+0000","last_fullsized":"2024-03-29T05:10:26.865624+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:39:31.307835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":931,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.804572+0000","last_change":"2024-03-29T04:59:33.401536+0000","last_active":"2024-03-29T05:10:22.804572+0000","last_peered":"2024-03-29T05:10:22.804572+0000","last_clean":"2024-03-29T05:10:22.804572+0000","last_became_active":"2024-03-29T04:59:33.401119+0000","last_became_peered":"2024-03-29T04:59:33.401119+0000","last_unstale":"2024-03-29T05:10:22.804572+0000","last_undegraded":"2024-03-29T05:10:22.804572+0000","last_fullsized":"2024-03-29T05:10:22.804572+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:38:47.459663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1447,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.802291+0000","last_change":"2024-03-29T05:01:53.826079+0000","last_active":"2024-03-29T05:10:26.802291+0000","last_peered":"2024-03-29T05:10:26.802291+0000","last_clean":"2024-03-29T05:10:26.802291+0000","last_became_active":"2024-03-29T05:01:53.825729+0000","last_became_peered":"2024-03-29T05:01:53.825729+0000","last_unstale":"2024-03-29T05:10:26.802291+0000","last_undegraded":"2024-03-29T05:10:26.802291+0000","last_fullsized":"2024-03-29T05:10:26.802291+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:08:51.102523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":920,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.090140+0000","last_change":"2024-03-29T05:02:15.266708+0000","last_active":"2024-03-29T05:10:23.090140+0000","last_peered":"2024-03-29T05:10:23.090140+0000","last_clean":"2024-03-29T05:10:23.090140+0000","last_became_active":"2024-03-29T05:02:15.266130+0000","last_became_peered":"2024-03-29T05:02:15.266130+0000","last_unstale":"2024-03-29T05:10:23.090140+0000","last_undegraded":"2024-03-29T05:10:23.090140+0000","last_fullsized":"2024-03-29T05:10:23.090140+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:45:27.270692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":925,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.731071+0000","last_change":"2024-03-29T05:01:53.826799+0000","last_active":"2024-03-29T05:10:22.731071+0000","last_peered":"2024-03-29T05:10:22.731071+0000","last_clean":"2024-03-29T05:10:22.731071+0000","last_became_active":"2024-03-29T05:01:53.826480+0000","last_became_peered":"2024-03-29T05:01:53.826480+0000","last_unstale":"2024-03-29T05:10:22.731071+0000","last_undegraded":"2024-03-29T05:10:22.731071+0000","last_fullsized":"2024-03-29T05:10:22.731071+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:41:19.898891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1447,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.801561+0000","last_change":"2024-03-29T05:01:53.825119+0000","last_active":"2024-03-29T05:10:26.801561+0000","last_peered":"2024-03-29T05:10:26.801561+0000","last_clean":"2024-03-29T05:10:26.801561+0000","last_became_active":"2024-03-29T05:01:53.824758+0000","last_became_peered":"2024-03-29T05:01:53.824758+0000","last_unstale":"2024-03-29T05:10:26.801561+0000","last_undegraded":"2024-03-29T05:10:26.801561+0000","last_fullsized":"2024-03-29T05:10:26.801561+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:10:23.329902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1331,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.865263+0000","last_change":"2024-03-29T05:01:53.829740+0000","last_active":"2024-03-29T05:10:26.865263+0000","last_peered":"2024-03-29T05:10:26.865263+0000","last_clean":"2024-03-29T05:10:26.865263+0000","last_became_active":"2024-03-29T05:01:53.829454+0000","last_became_peered":"2024-03-29T05:01:53.829454+0000","last_unstale":"2024-03-29T05:10:26.865263+0000","last_undegraded":"2024-03-29T05:10:26.865263+0000","last_fullsized":"2024-03-29T05:10:26.865263+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:14:31.335348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":939,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.090418+0000","last_change":"2024-03-29T05:02:15.266831+0000","last_active":"2024-03-29T05:10:23.090418+0000","last_peered":"2024-03-29T05:10:23.090418+0000","last_clean":"2024-03-29T05:10:23.090418+0000","last_became_active":"2024-03-29T05:02:15.266462+0000","last_became_peered":"2024-03-29T05:02:15.266462+0000","last_unstale":"2024-03-29T05:10:23.090418+0000","last_undegraded":"2024-03-29T05:10:23.090418+0000","last_fullsized":"2024-03-29T05:10:23.090418+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:26:10.471305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1298,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.045436+0000","last_change":"2024-03-29T05:02:15.264444+0000","last_active":"2024-03-29T05:10:26.045436+0000","last_peered":"2024-03-29T05:10:26.045436+0000","last_clean":"2024-03-29T05:10:26.045436+0000","last_became_active":"2024-03-29T05:02:15.260792+0000","last_became_peered":"2024-03-29T05:02:15.260792+0000","last_unstale":"2024-03-29T05:10:26.045436+0000","last_undegraded":"2024-03-29T05:10:26.045436+0000","last_fullsized":"2024-03-29T05:10:26.045436+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:16:34.226178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"329'22","reported_seq":1305,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.044574+0000","last_change":"2024-03-29T05:02:15.262266+0000","last_active":"2024-03-29T05:10:26.044574+0000","last_peered":"2024-03-29T05:10:26.044574+0000","last_clean":"2024-03-29T05:10:26.044574+0000","last_became_active":"2024-03-29T05:02:15.261863+0000","last_became_peered":"2024-03-29T05:02:15.261863+0000","last_unstale":"2024-03-29T05:10:26.044574+0000","last_undegraded":"2024-03-29T05:10:26.044574+0000","last_fullsized":"2024-03-29T05:10:26.044574+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:08:24.941726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1398,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.084671+0000","last_change":"2024-03-29T05:01:18.509387+0000","last_active":"2024-03-29T05:10:26.084671+0000","last_peered":"2024-03-29T05:10:26.084671+0000","last_clean":"2024-03-29T05:10:26.084671+0000","last_became_active":"2024-03-29T05:01:18.509063+0000","last_became_peered":"2024-03-29T05:01:18.509063+0000","last_unstale":"2024-03-29T05:10:26.084671+0000","last_undegraded":"2024-03-29T05:10:26.084671+0000","last_fullsized":"2024-03-29T05:10:26.084671+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:31:04.765878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":940,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:23.090782+0000","last_change":"2024-03-29T05:02:15.260968+0000","last_active":"2024-03-29T05:10:23.090782+0000","last_peered":"2024-03-29T05:10:23.090782+0000","last_clean":"2024-03-29T05:10:23.090782+0000","last_became_active":"2024-03-29T05:02:15.260316+0000","last_became_peered":"2024-03-29T05:02:15.260316+0000","last_unstale":"2024-03-29T05:10:23.090782+0000","last_undegraded":"2024-03-29T05:10:23.090782+0000","last_fullsized":"2024-03-29T05:10:23.090782+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:46:24.566561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1301,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.865588+0000","last_change":"2024-03-29T05:01:53.843008+0000","last_active":"2024-03-29T05:10:26.865588+0000","last_peered":"2024-03-29T05:10:26.865588+0000","last_clean":"2024-03-29T05:10:26.865588+0000","last_became_active":"2024-03-29T05:01:53.842627+0000","last_became_peered":"2024-03-29T05:01:53.842627+0000","last_unstale":"2024-03-29T05:10:26.865588+0000","last_undegraded":"2024-03-29T05:10:26.865588+0000","last_fullsized":"2024-03-29T05:10:26.865588+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:54:18.788660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":926,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.730729+0000","last_change":"2024-03-29T05:00:43.914390+0000","last_active":"2024-03-29T05:10:22.730729+0000","last_peered":"2024-03-29T05:10:22.730729+0000","last_clean":"2024-03-29T05:10:22.730729+0000","last_became_active":"2024-03-29T05:00:43.914064+0000","last_became_peered":"2024-03-29T05:00:43.914064+0000","last_unstale":"2024-03-29T05:10:22.730729+0000","last_undegraded":"2024-03-29T05:10:22.730729+0000","last_fullsized":"2024-03-29T05:10:22.730729+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:39:51.614460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1331,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.864685+0000","last_change":"2024-03-29T05:01:53.840294+0000","last_active":"2024-03-29T05:10:26.864685+0000","last_peered":"2024-03-29T05:10:26.864685+0000","last_clean":"2024-03-29T05:10:26.864685+0000","last_became_active":"2024-03-29T05:01:53.840016+0000","last_became_peered":"2024-03-29T05:01:53.840016+0000","last_unstale":"2024-03-29T05:10:26.864685+0000","last_undegraded":"2024-03-29T05:10:26.864685+0000","last_fullsized":"2024-03-29T05:10:26.864685+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:06:30.265575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1446,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.802461+0000","last_change":"2024-03-29T05:01:53.822039+0000","last_active":"2024-03-29T05:10:26.802461+0000","last_peered":"2024-03-29T05:10:26.802461+0000","last_clean":"2024-03-29T05:10:26.802461+0000","last_became_active":"2024-03-29T05:01:53.821657+0000","last_became_peered":"2024-03-29T05:01:53.821657+0000","last_unstale":"2024-03-29T05:10:26.802461+0000","last_undegraded":"2024-03-29T05:10:26.802461+0000","last_fullsized":"2024-03-29T05:10:26.802461+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:00:15.680313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":922,"reported_epoch":971,"state":"active+clean","last_fresh":"2024-03-29T05:10:22.804997+0000","last_change":"2024-03-29T05:00:43.916081+0000","last_active":"2024-03-29T05:10:22.804997+0000","last_peered":"2024-03-29T05:10:22.804997+0000","last_clean":"2024-03-29T05:10:22.804997+0000","last_became_active":"2024-03-29T05:00:43.915222+0000","last_became_peered":"2024-03-29T05:00:43.915222+0000","last_unstale":"2024-03-29T05:10:22.804997+0000","last_undegraded":"2024-03-29T05:10:22.804997+0000","last_fullsized":"2024-03-29T05:10:22.804997+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:23:17.449789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1348,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.864627+0000","last_change":"2024-03-29T05:01:53.826966+0000","last_active":"2024-03-29T05:10:26.864627+0000","last_peered":"2024-03-29T05:10:26.864627+0000","last_clean":"2024-03-29T05:10:26.864627+0000","last_became_active":"2024-03-29T05:01:53.826460+0000","last_became_peered":"2024-03-29T05:01:53.826460+0000","last_unstale":"2024-03-29T05:10:26.864627+0000","last_undegraded":"2024-03-29T05:10:26.864627+0000","last_fullsized":"2024-03-29T05:10:26.864627+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:36:08.169321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1448,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.801804+0000","last_change":"2024-03-29T05:01:53.828459+0000","last_active":"2024-03-29T05:10:26.801804+0000","last_peered":"2024-03-29T05:10:26.801804+0000","last_clean":"2024-03-29T05:10:26.801804+0000","last_became_active":"2024-03-29T05:01:53.828149+0000","last_became_peered":"2024-03-29T05:01:53.828149+0000","last_unstale":"2024-03-29T05:10:26.801804+0000","last_undegraded":"2024-03-29T05:10:26.801804+0000","last_fullsized":"2024-03-29T05:10:26.801804+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:49:51.889159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1407,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.084617+0000","last_change":"2024-03-29T05:02:15.249825+0000","last_active":"2024-03-29T05:10:26.084617+0000","last_peered":"2024-03-29T05:10:26.084617+0000","last_clean":"2024-03-29T05:10:26.084617+0000","last_became_active":"2024-03-29T05:02:15.249461+0000","last_became_peered":"2024-03-29T05:02:15.249461+0000","last_unstale":"2024-03-29T05:10:26.084617+0000","last_undegraded":"2024-03-29T05:10:26.084617+0000","last_fullsized":"2024-03-29T05:10:26.084617+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:24:19.894615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1342,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.865151+0000","last_change":"2024-03-29T05:02:15.258674+0000","last_active":"2024-03-29T05:10:26.865151+0000","last_peered":"2024-03-29T05:10:26.865151+0000","last_clean":"2024-03-29T05:10:26.865151+0000","last_became_active":"2024-03-29T05:02:15.257963+0000","last_became_peered":"2024-03-29T05:02:15.257963+0000","last_unstale":"2024-03-29T05:10:26.865151+0000","last_undegraded":"2024-03-29T05:10:26.865151+0000","last_fullsized":"2024-03-29T05:10:26.865151+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:50.869724+0000","last_clean_scrub_stamp":"2024-03-29T04:51:50.869724+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:29:43.805122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":1310,"reported_epoch":972,"state":"active+clean","last_fresh":"2024-03-29T05:10:26.045027+0000","last_change":"2024-03-29T05:02:15.250743+0000","last_active":"2024-03-29T05:10:26.045027+0000","last_peered":"2024-03-29T05:10:26.045027+0000","last_clean":"2024-03-29T05:10:26.045027+0000","last_became_active":"2024-03-29T05:02:15.250374+0000","last_became_peered":"2024-03-29T05:02:15.250374+0000","last_unstale":"2024-03-29T05:10:26.045027+0000","last_undegraded":"2024-03-29T05:10:26.045027+0000","last_fullsized":"2024-03-29T05:10:26.045027+0000","mapping_epoch":576,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":577,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:51:49.864892+0000","last_clean_scrub_stamp":"2024-03-29T04:51:49.864892+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:02:27.429005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2052,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201537,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":322256,"kb_used_data":8516,"kb_used_omap":1,"kb_used_meta":313726,"kb_avail":93422896,"statfs":{"total":95995035648,"available":95665045504,"internally_reserved":0,"allocated":8720384,"data_stored":6866451,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":321255768},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463175,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":318144,"kb_used_data":8504,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93427008,"statfs":{"total":95995035648,"available":95669256192,"internally_reserved":0,"allocated":8708096,"data_stored":6866305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":490,"seq":2104533975168,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":59392,"kb_used_data":9456,"kb_used_omap":53,"kb_used_meta":49866,"kb_avail":93685760,"statfs":{"total":95995035648,"available":95934218240,"internally_reserved":0,"allocated":9682944,"data_stored":6500300,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55167,"internal_metadata":51062913},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":425,"seq":1825361100942,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":55712,"kb_used_data":10068,"kb_used_omap":42,"kb_used_meta":45589,"kb_avail":93689440,"statfs":{"total":95995035648,"available":95937986560,"internally_reserved":0,"allocated":10309632,"data_stored":7096084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43631,"internal_metadata":46683537},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":525,"seq":2254857830521,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":63784,"kb_used_data":9628,"kb_used_omap":64,"kb_used_meta":54079,"kb_avail":93681368,"statfs":{"total":95995035648,"available":95929720832,"internally_reserved":0,"allocated":9859072,"data_stored":6526557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":66188,"internal_metadata":55377268},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":555,"seq":2383706849394,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":59232,"kb_used_data":9364,"kb_used_omap":55,"kb_used_meta":49800,"kb_avail":93685920,"statfs":{"total":95995035648,"available":95934382080,"internally_reserved":0,"allocated":9588736,"data_stored":6489166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56870,"internal_metadata":50995674},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":576,"seq":2473901162605,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":59364,"kb_used_data":9560,"kb_used_omap":56,"kb_used_meta":49735,"kb_avail":93685788,"statfs":{"total":95995035648,"available":95934246912,"internally_reserved":0,"allocated":9789440,"data_stored":6515747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58303,"internal_metadata":50928705},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724811,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":317560,"kb_used_data":7916,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93427592,"statfs":{"total":95995035648,"available":95669854208,"internally_reserved":0,"allocated":8105984,"data_stored":6271763,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":706,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1346,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T05:11:15.110 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-29T05:11:15.110 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-29T05:11:15.110 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-29T05:11:15.110 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:11:15.367 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:11:15.782 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:15 smithi181 ceph-mon[106466]: from='client.34890 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T05:11:15.782 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:15 smithi181 ceph-mon[106466]: pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:15.782 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:15 smithi181 ceph-mon[101970]: from='client.34890 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T05:11:15.782 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:15 smithi181 ceph-mon[101970]: pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:16.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:15 smithi187 ceph-mon[93536]: from='client.34890 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T05:11:16.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:15 smithi187 ceph-mon[93536]: pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:17.316 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:11:17.317 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:11:17.930 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:17 smithi181 ceph-mon[106466]: pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:17.930 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:17 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/14952101' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:17.930 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:17 smithi181 ceph-mon[101970]: pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:17.930 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:17 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/14952101' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:18.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:17 smithi187 ceph-mon[93536]: pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:18.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:17 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/14952101' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:19.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:11:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:11:18] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.43.0" 2024-03-29T05:11:19.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:19 smithi181 ceph-mon[106466]: pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:19.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:19 smithi181 ceph-mon[101970]: pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:20.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:19 smithi187 ceph-mon[93536]: pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:20.955 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:11:21.224 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:11:21.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:21 smithi181 ceph-mon[106466]: pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:21.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:21 smithi181 ceph-mon[101970]: pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:22.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:21 smithi187 ceph-mon[93536]: pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:23.085 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:11:23.085 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:11:23.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:23 smithi181 ceph-mon[101970]: pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:23.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:23 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1790848361' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:23.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:23 smithi181 ceph-mon[106466]: pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:23.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:23 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1790848361' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:24.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:23 smithi187 ceph-mon[93536]: pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:24.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:23 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1790848361' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:26.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:25 smithi187 ceph-mon[93536]: pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:26.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:25 smithi181 ceph-mon[106466]: pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:26.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:25 smithi181 ceph-mon[101970]: pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:26.722 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:11:26.989 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:11:27.111 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:11:27.111 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:11:27.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:11:28.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:27 smithi187 ceph-mon[93536]: pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:28.152 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:27 smithi181 ceph-mon[106466]: pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:28.153 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:27 smithi181 ceph-mon[101970]: pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:28.908 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:11:28.909 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:11:29.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:28 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:11:29.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:28 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:11:29.203 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:28 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:11:29.203 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:11:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:11:28] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.43.0" 2024-03-29T05:11:29.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:29 smithi187 ceph-mon[93536]: pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:29.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:29 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/639498136' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:29.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:29 smithi181 ceph-mon[101970]: pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:29.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:29 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/639498136' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:29.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:29 smithi181 ceph-mon[106466]: pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:29.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:29 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/639498136' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:31.842 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:31 smithi187 ceph-mon[93536]: pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:32.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:31 smithi181 ceph-mon[106466]: pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:32.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:31 smithi181 ceph-mon[101970]: pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:32.527 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:11:32.925 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:11:33.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:33 smithi181 ceph-mon[101970]: pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:33.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:33 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:11:33.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:33 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:11:33.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:33 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:11:33.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:33 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:11:33.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:33 smithi181 ceph-mon[106466]: pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:33.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:33 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:11:33.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:33 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:11:33.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:33 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:11:33.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:33 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:11:34.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:33 smithi187 ceph-mon[93536]: pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:34.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:33 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:11:34.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:33 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:11:34.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:33 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:11:34.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:33 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:11:34.845 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:11:34.846 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:11:34.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:34 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:11:34.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:34 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:11:34.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:34 smithi181 ceph-mon[101970]: pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:34.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:34 smithi181 ceph-mon[101970]: pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:34.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:34 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:11:34.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:34 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:11:34.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:34 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:11:34.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:34 smithi181 ceph-mon[106466]: pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:34.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:34 smithi181 ceph-mon[106466]: pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:34.953 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:34 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:11:35.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:34 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:11:35.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:34 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:11:35.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:34 smithi187 ceph-mon[93536]: pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:35.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:34 smithi187 ceph-mon[93536]: pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:35.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:34 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:11:35.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:35 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2975437552' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:35.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:35 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2975437552' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:36.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:35 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2975437552' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:36.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:36 smithi181 ceph-mon[101970]: pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:36.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:36 smithi181 ceph-mon[106466]: pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:37.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:36 smithi187 ceph-mon[93536]: pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:38.441 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:11:38.706 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:11:39.111 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:38 smithi181 ceph-mon[106466]: pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:39.111 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:38 smithi181 ceph-mon[101970]: pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:39.112 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:11:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:11:38] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.43.0" 2024-03-29T05:11:39.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:38 smithi187 ceph-mon[93536]: pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:40.619 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:11:40.619 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:11:41.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:40 smithi187 ceph-mon[93536]: pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:41.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:40 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/840072948' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:41.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:40 smithi181 ceph-mon[101970]: pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:41.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:40 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/840072948' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:41.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:40 smithi181 ceph-mon[106466]: pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:41.198 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:40 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/840072948' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:42.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:11:42.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:11:42.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:11:43.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:42 smithi187 ceph-mon[93536]: pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:43.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:42 smithi181 ceph-mon[106466]: pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:43.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:42 smithi181 ceph-mon[101970]: pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:44.223 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:11:44.491 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:11:45.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:44 smithi187 ceph-mon[93536]: pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:45.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:44 smithi181 ceph-mon[106466]: pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:45.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:44 smithi181 ceph-mon[101970]: pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:46.489 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:11:46.489 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:11:47.061 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:46 smithi181 ceph-mon[106466]: pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:47.062 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:46 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1692393837' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:47.062 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:46 smithi181 ceph-mon[101970]: pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:47.062 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:46 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1692393837' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:47.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:46 smithi187 ceph-mon[93536]: pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:47.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:46 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1692393837' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:49.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:48 smithi181 ceph-mon[106466]: pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:49.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:48 smithi181 ceph-mon[101970]: pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:49.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:11:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:11:48] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-03-29T05:11:49.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:48 smithi187 ceph-mon[93536]: pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:50.087 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:11:50.354 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:11:51.193 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:50 smithi181 ceph-mon[101970]: pgmap v1235: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:51.194 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:50 smithi181 ceph-mon[106466]: pgmap v1235: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:51.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:50 smithi187 ceph-mon[93536]: pgmap v1235: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:52.290 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:11:52.290 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:11:52.926 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:52 smithi181 ceph-mon[101970]: pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:52.926 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:52 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1257400737' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:52.927 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:52 smithi181 ceph-mon[106466]: pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:52.927 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:52 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1257400737' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:53.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:52 smithi187 ceph-mon[93536]: pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:53.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:52 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1257400737' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:55.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:54 smithi181 ceph-mon[101970]: pgmap v1237: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:55.300 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:54 smithi181 ceph-mon[106466]: pgmap v1237: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:55.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:54 smithi187 ceph-mon[93536]: pgmap v1237: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:55.953 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:11:56.220 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:11:57.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:56 smithi181 ceph-mon[101970]: pgmap v1238: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:57.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:11:57.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:56 smithi181 ceph-mon[106466]: pgmap v1238: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:57.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:11:57.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:56 smithi187 ceph-mon[93536]: pgmap v1238: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:57.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:11:58.133 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:11:58.133 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:11:59.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:11:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:11:58] "GET /metrics HTTP/1.1" 200 34117 "" "Prometheus/2.43.0" 2024-03-29T05:11:59.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:58 smithi181 ceph-mon[106466]: pgmap v1239: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:59.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:11:58 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3909392809' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:59.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:58 smithi181 ceph-mon[101970]: pgmap v1239: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:59.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:11:58 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3909392809' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:11:59.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:58 smithi187 ceph-mon[93536]: pgmap v1239: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:11:59.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:11:58 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3909392809' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:01.184 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:00 smithi187 ceph-mon[93536]: pgmap v1240: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:01.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:00 smithi181 ceph-mon[101970]: pgmap v1240: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:01.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:00 smithi181 ceph-mon[106466]: pgmap v1240: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:01.757 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:12:02.028 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:12:03.172 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:02 smithi181 ceph-mon[106466]: pgmap v1241: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:03.173 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:02 smithi181 ceph-mon[101970]: pgmap v1241: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:03.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:02 smithi187 ceph-mon[93536]: pgmap v1241: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:03.971 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:12:03.972 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:12:05.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:04 smithi181 ceph-mon[101970]: pgmap v1242: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:05.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:04 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3175327468' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:05.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:04 smithi181 ceph-mon[106466]: pgmap v1242: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:05.203 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:04 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3175327468' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:05.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:04 smithi187 ceph-mon[93536]: pgmap v1242: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:05.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:04 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3175327468' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:07.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:06 smithi181 ceph-mon[101970]: pgmap v1243: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:07.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:06 smithi181 ceph-mon[106466]: pgmap v1243: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:07.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:06 smithi187 ceph-mon[93536]: pgmap v1243: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:07.596 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:12:07.863 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:12:08.996 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:12:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:12:08] "GET /metrics HTTP/1.1" 200 34117 "" "Prometheus/2.43.0" 2024-03-29T05:12:08.996 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:08 smithi181 ceph-mon[106466]: pgmap v1244: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:08.996 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:08 smithi181 ceph-mon[101970]: pgmap v1244: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:09.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:08 smithi187 ceph-mon[93536]: pgmap v1244: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:09.739 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:12:09.739 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:12:10.120 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:09 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1880599147' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:10.120 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:09 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1880599147' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:10.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:09 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1880599147' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:11.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:10 smithi181 ceph-mon[101970]: pgmap v1245: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:11.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:10 smithi181 ceph-mon[106466]: pgmap v1245: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:11.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:10 smithi187 ceph-mon[93536]: pgmap v1245: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:12.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:12:12.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:12:12.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:12:13.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:12 smithi181 ceph-mon[101970]: pgmap v1246: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:13.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:12 smithi181 ceph-mon[106466]: pgmap v1246: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:13.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:12 smithi187 ceph-mon[93536]: pgmap v1246: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:13.454 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:12:13.726 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:12:15.156 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:14 smithi181 ceph-mon[106466]: pgmap v1247: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:15.157 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:14 smithi181 ceph-mon[101970]: pgmap v1247: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:15.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:14 smithi187 ceph-mon[93536]: pgmap v1247: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:15.645 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:12:15.646 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:12:16.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:15 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1276655125' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:16.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:15 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1276655125' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:16.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:15 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1276655125' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:17.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:16 smithi181 ceph-mon[106466]: pgmap v1248: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:17.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:16 smithi181 ceph-mon[101970]: pgmap v1248: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:17.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:16 smithi187 ceph-mon[93536]: pgmap v1248: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:19.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:18 smithi181 ceph-mon[106466]: pgmap v1249: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:19.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:18 smithi181 ceph-mon[101970]: pgmap v1249: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:19.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:12:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:12:18] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-03-29T05:12:19.309 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:12:19.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:18 smithi187 ceph-mon[93536]: pgmap v1249: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:19.577 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:12:21.027 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:20 smithi181 ceph-mon[101970]: pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:21.028 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:20 smithi181 ceph-mon[106466]: pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:21.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:20 smithi187 ceph-mon[93536]: pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:21.516 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:12:21.517 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:12:22.124 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:21 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/4240857276' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:22.124 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:21 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/4240857276' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:22.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:21 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/4240857276' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:23.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:22 smithi181 ceph-mon[106466]: pgmap v1251: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:23.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:22 smithi181 ceph-mon[101970]: pgmap v1251: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:23.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:22 smithi187 ceph-mon[93536]: pgmap v1251: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:25.148 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:12:25.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:24 smithi181 ceph-mon[106466]: pgmap v1252: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:25.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:24 smithi181 ceph-mon[101970]: pgmap v1252: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:25.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:24 smithi187 ceph-mon[93536]: pgmap v1252: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:25.414 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:12:27.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:26 smithi181 ceph-mon[101970]: pgmap v1253: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:27.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:12:27.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:26 smithi181 ceph-mon[106466]: pgmap v1253: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:27.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:12:27.305 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:12:27.305 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:12:27.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:26 smithi187 ceph-mon[93536]: pgmap v1253: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:27.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:12:28.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:27 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1127529291' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:28.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:27 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1127529291' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:28.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:27 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1127529291' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:29.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:28 smithi187 ceph-mon[93536]: pgmap v1254: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:29.452 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:28 smithi181 ceph-mon[106466]: pgmap v1254: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:29.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:28 smithi181 ceph-mon[101970]: pgmap v1254: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:29.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:12:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:12:28] "GET /metrics HTTP/1.1" 200 34117 "" "Prometheus/2.43.0" 2024-03-29T05:12:30.971 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:12:31.234 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:12:31.356 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:30 smithi181 ceph-mon[101970]: pgmap v1255: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:31.356 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:30 smithi181 ceph-mon[106466]: pgmap v1255: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:31.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:30 smithi187 ceph-mon[93536]: pgmap v1255: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:33.133 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:12:33.133 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:12:33.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:32 smithi187 ceph-mon[93536]: pgmap v1256: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:33.451 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:32 smithi181 ceph-mon[106466]: pgmap v1256: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:33.452 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:32 smithi181 ceph-mon[101970]: pgmap v1256: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:34.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:33 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2634306934' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:34.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:33 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:12:34.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:33 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2634306934' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:34.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:33 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:12:34.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:33 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2634306934' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:34.382 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:33 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:12:35.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:34 smithi187 ceph-mon[93536]: pgmap v1257: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:35.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:34 smithi181 ceph-mon[101970]: pgmap v1257: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:35.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:34 smithi181 ceph-mon[106466]: pgmap v1257: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:36.830 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:12:37.199 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:36 smithi181 ceph-mon[106466]: pgmap v1258: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:37.199 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:36 smithi181 ceph-mon[101970]: pgmap v1258: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:37.333 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:36 smithi187 ceph-mon[93536]: pgmap v1258: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:37.659 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:12:38.825 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:12:38.826 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:12:38.826 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:12:38.826 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:12:38.826 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-mon[106466]: pgmap v1259: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:38.826 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:12:38.826 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:12:38.826 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:12:38.826 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:12:38.826 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:12:38.826 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:12:38.826 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:12:38.827 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-mon[101970]: pgmap v1259: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:38.827 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:12:38.827 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:12:38.827 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:12:39.082 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:12:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:12:38] "GET /metrics HTTP/1.1" 200 34117 "" "Prometheus/2.43.0" 2024-03-29T05:12:39.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:12:39.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:12:39.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:12:39.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:12:39.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:38 smithi187 ceph-mon[93536]: pgmap v1259: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:39.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:12:39.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:12:39.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:12:39.618 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:12:39.618 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:12:39.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:39 smithi181 ceph-mon[101970]: pgmap v1260: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:39.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:39 smithi181 ceph-mon[101970]: pgmap v1261: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:39.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:39 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/4223832834' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:39.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:39 smithi181 ceph-mon[106466]: pgmap v1260: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:39.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:39 smithi181 ceph-mon[106466]: pgmap v1261: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:39.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:39 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/4223832834' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:40.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:39 smithi187 ceph-mon[93536]: pgmap v1260: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:40.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:39 smithi187 ceph-mon[93536]: pgmap v1261: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:40.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:39 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/4223832834' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:42.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:41 smithi187 ceph-mon[93536]: pgmap v1262: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:42.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:12:42.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:41 smithi181 ceph-mon[101970]: pgmap v1262: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:42.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:12:42.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:41 smithi181 ceph-mon[106466]: pgmap v1262: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:42.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:12:43.250 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:12:43.519 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:12:43.944 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:43 smithi181 ceph-mon[106466]: pgmap v1263: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:43.945 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:43 smithi181 ceph-mon[101970]: pgmap v1263: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:44.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:43 smithi187 ceph-mon[93536]: pgmap v1263: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:45.422 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:12:45.423 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:12:46.057 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:45 smithi181 ceph-mon[101970]: pgmap v1264: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:46.058 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:45 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/271942283' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:46.058 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:45 smithi181 ceph-mon[106466]: pgmap v1264: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:46.058 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:45 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/271942283' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:46.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:45 smithi187 ceph-mon[93536]: pgmap v1264: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:46.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:45 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/271942283' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:48.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:47 smithi187 ceph-mon[93536]: pgmap v1265: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:48.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:47 smithi181 ceph-mon[101970]: pgmap v1265: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:48.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:47 smithi181 ceph-mon[106466]: pgmap v1265: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:49.083 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:12:49.348 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:12:49.451 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:12:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:12:48] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2024-03-29T05:12:50.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:49 smithi187 ceph-mon[93536]: pgmap v1266: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:50.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:49 smithi181 ceph-mon[106466]: pgmap v1266: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:50.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:49 smithi181 ceph-mon[101970]: pgmap v1266: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:51.222 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:12:51.223 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:12:52.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:51 smithi187 ceph-mon[93536]: pgmap v1267: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:52.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:51 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3917662082' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:52.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:51 smithi181 ceph-mon[106466]: pgmap v1267: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:52.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:51 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3917662082' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:52.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:51 smithi181 ceph-mon[101970]: pgmap v1267: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:52.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:51 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3917662082' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:54.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:53 smithi187 ceph-mon[93536]: pgmap v1268: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:54.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:53 smithi181 ceph-mon[101970]: pgmap v1268: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:54.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:53 smithi181 ceph-mon[106466]: pgmap v1268: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:54.795 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:12:55.061 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:12:56.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:55 smithi187 ceph-mon[93536]: pgmap v1269: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:56.169 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:55 smithi181 ceph-mon[101970]: pgmap v1269: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:56.169 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:55 smithi181 ceph-mon[106466]: pgmap v1269: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:56.920 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:12:56.920 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:12:57.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:12:57.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:12:57.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:12:57.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:57 smithi181 ceph-mon[106466]: pgmap v1270: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:57.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:57 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1579883738' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:57.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:57 smithi181 ceph-mon[101970]: pgmap v1270: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:57.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:57 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1579883738' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:58.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:57 smithi187 ceph-mon[93536]: pgmap v1270: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:12:58.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:57 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1579883738' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:12:59.451 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:12:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:12:58] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2024-03-29T05:13:00.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:12:59 smithi187 ceph-mon[93536]: pgmap v1271: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:00.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:12:59 smithi181 ceph-mon[101970]: pgmap v1271: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:00.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:12:59 smithi181 ceph-mon[106466]: pgmap v1271: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:00.602 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:13:00.870 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:13:02.067 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:01 smithi181 ceph-mon[101970]: pgmap v1272: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:02.067 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:01 smithi181 ceph-mon[106466]: pgmap v1272: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:02.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:01 smithi187 ceph-mon[93536]: pgmap v1272: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:02.828 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:13:02.828 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:13:03.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:03 smithi181 ceph-mon[101970]: pgmap v1273: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:03.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:03 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/713079308' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:03.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:03 smithi181 ceph-mon[106466]: pgmap v1273: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:03.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:03 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/713079308' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:04.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:03 smithi187 ceph-mon[93536]: pgmap v1273: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:04.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:03 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/713079308' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:06.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:05 smithi187 ceph-mon[93536]: pgmap v1274: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:06.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:05 smithi181 ceph-mon[106466]: pgmap v1274: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:06.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:05 smithi181 ceph-mon[101970]: pgmap v1274: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:06.633 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:13:06.901 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:13:08.082 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:07 smithi181 ceph-mon[106466]: pgmap v1275: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:08.082 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:07 smithi181 ceph-mon[101970]: pgmap v1275: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:08.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:07 smithi187 ceph-mon[93536]: pgmap v1275: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:08.870 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:13:08.870 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:13:09.198 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:13:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:13:08] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2024-03-29T05:13:10.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:09 smithi187 ceph-mon[93536]: pgmap v1276: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:10.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:09 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2015659479' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:10.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:09 smithi181 ceph-mon[106466]: pgmap v1276: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:10.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:09 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2015659479' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:10.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:09 smithi181 ceph-mon[101970]: pgmap v1276: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:10.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:09 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2015659479' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:12.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:11 smithi187 ceph-mon[93536]: pgmap v1277: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:12.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:13:12.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:11 smithi181 ceph-mon[106466]: pgmap v1277: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:12.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:13:12.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:11 smithi181 ceph-mon[101970]: pgmap v1277: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:12.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:13:12.515 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:13:12.781 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:13:13.854 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:13 smithi181 ceph-mon[106466]: pgmap v1278: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:13.855 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:13 smithi181 ceph-mon[101970]: pgmap v1278: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:14.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:13 smithi187 ceph-mon[93536]: pgmap v1278: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:14.614 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:13:14.614 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:13:14.944 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:14 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2367253825' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:14.944 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:14 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2367253825' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:15.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:14 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2367253825' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:16.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:15 smithi187 ceph-mon[93536]: pgmap v1279: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:16.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:15 smithi181 ceph-mon[101970]: pgmap v1279: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:16.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:15 smithi181 ceph-mon[106466]: pgmap v1279: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:18.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:17 smithi187 ceph-mon[93536]: pgmap v1280: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:18.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:17 smithi181 ceph-mon[101970]: pgmap v1280: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:18.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:17 smithi181 ceph-mon[106466]: pgmap v1280: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:18.244 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:13:18.511 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:13:19.201 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:13:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:13:18] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.43.0" 2024-03-29T05:13:20.106 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:19 smithi181 ceph-mon[101970]: pgmap v1281: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:20.106 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:19 smithi181 ceph-mon[106466]: pgmap v1281: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:20.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:19 smithi187 ceph-mon[93536]: pgmap v1281: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:20.448 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:13:20.448 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:13:21.034 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:20 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/117011252' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:21.034 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:20 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/117011252' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:21.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:20 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/117011252' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:22.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:21 smithi187 ceph-mon[93536]: pgmap v1282: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:22.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:21 smithi181 ceph-mon[106466]: pgmap v1282: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:22.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:21 smithi181 ceph-mon[101970]: pgmap v1282: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:24.058 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:13:24.089 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:23 smithi181 ceph-mon[101970]: pgmap v1283: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:24.089 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:23 smithi181 ceph-mon[106466]: pgmap v1283: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:24.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:23 smithi187 ceph-mon[93536]: pgmap v1283: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:24.325 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:13:26.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:25 smithi187 ceph-mon[93536]: pgmap v1284: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:26.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:25 smithi181 ceph-mon[106466]: pgmap v1284: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:26.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:25 smithi181 ceph-mon[101970]: pgmap v1284: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:26.234 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:13:26.234 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:13:27.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:26 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1529889940' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:27.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:13:27.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:26 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1529889940' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:27.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:13:27.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:26 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1529889940' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:27.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:13:28.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:27 smithi187 ceph-mon[93536]: pgmap v1285: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:28.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:27 smithi181 ceph-mon[101970]: pgmap v1285: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:28.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:27 smithi181 ceph-mon[106466]: pgmap v1285: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:29.451 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:13:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:13:28] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.43.0" 2024-03-29T05:13:29.807 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:13:30.073 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:13:30.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:29 smithi187 ceph-mon[93536]: pgmap v1286: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:30.196 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:29 smithi181 ceph-mon[101970]: pgmap v1286: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:30.196 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:29 smithi181 ceph-mon[106466]: pgmap v1286: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:31.906 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:13:31.906 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:13:32.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:31 smithi187 ceph-mon[93536]: pgmap v1287: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:32.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:31 smithi181 ceph-mon[101970]: pgmap v1287: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:32.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:31 smithi181 ceph-mon[106466]: pgmap v1287: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:33.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:32 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2153947042' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:33.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:32 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2153947042' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:33.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:32 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2153947042' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:34.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:33 smithi187 ceph-mon[93536]: pgmap v1288: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:34.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:33 smithi181 ceph-mon[101970]: pgmap v1288: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:34.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:33 smithi181 ceph-mon[106466]: pgmap v1288: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:35.470 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:13:35.737 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:13:36.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:35 smithi187 ceph-mon[93536]: pgmap v1289: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:36.139 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:35 smithi181 ceph-mon[101970]: pgmap v1289: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:36.139 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:35 smithi181 ceph-mon[106466]: pgmap v1289: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:37.543 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:13:37.543 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:13:38.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:37 smithi187 ceph-mon[93536]: pgmap v1290: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:38.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:37 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2166667545' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:38.158 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:37 smithi181 ceph-mon[101970]: pgmap v1290: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:38.159 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:37 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2166667545' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:38.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:37 smithi181 ceph-mon[106466]: pgmap v1290: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:38.159 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:37 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2166667545' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:39.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:38 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:13:39.201 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:13:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:13:38] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.43.0" 2024-03-29T05:13:39.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:38 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:13:39.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:38 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:13:40.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:39 smithi187 ceph-mon[93536]: pgmap v1291: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:40.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:39 smithi181 ceph-mon[106466]: pgmap v1291: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:40.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:39 smithi181 ceph-mon[101970]: pgmap v1291: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:41.183 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:13:41.642 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:13:42.032 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:41 smithi181 ceph-mon[106466]: pgmap v1292: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:42.032 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:13:42.033 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:41 smithi181 ceph-mon[101970]: pgmap v1292: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:42.033 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:13:42.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:41 smithi187 ceph-mon[93536]: pgmap v1292: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:42.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:13:43.827 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:43 smithi181 ceph-mon[106466]: pgmap v1293: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:43.827 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:43 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:13:43.827 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:43 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:13:43.827 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:43 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:13:43.827 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:43 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:13:43.827 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:43 smithi181 ceph-mon[101970]: pgmap v1293: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:43.827 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:43 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:13:43.827 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:43 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:13:43.827 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:43 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:13:43.828 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:43 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:13:43.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:43 smithi187 ceph-mon[93536]: pgmap v1293: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:43.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:43 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:13:43.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:43 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:13:43.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:43 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:13:43.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:43 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:13:44.087 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:13:44.087 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:13:44.701 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:44 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:13:44.701 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:44 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:13:44.701 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:44 smithi181 ceph-mon[106466]: pgmap v1294: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:44.701 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:44 smithi181 ceph-mon[106466]: pgmap v1295: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:44.701 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:44 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:13:44.701 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:44 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/980620587' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:44.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:44 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:13:44.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:44 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:13:44.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:44 smithi181 ceph-mon[101970]: pgmap v1294: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:44.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:44 smithi181 ceph-mon[101970]: pgmap v1295: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:44.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:44 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:13:44.702 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:44 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/980620587' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:44.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:44 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:13:44.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:44 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:13:44.881 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:44 smithi187 ceph-mon[93536]: pgmap v1294: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:44.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:44 smithi187 ceph-mon[93536]: pgmap v1295: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:44.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:44 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:13:44.882 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:44 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/980620587' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:47.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:46 smithi187 ceph-mon[93536]: pgmap v1296: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:47.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:46 smithi181 ceph-mon[101970]: pgmap v1296: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:47.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:46 smithi181 ceph-mon[106466]: pgmap v1296: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:47.738 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:13:48.004 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:13:48.979 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:13:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:13:48] "GET /metrics HTTP/1.1" 200 34111 "" "Prometheus/2.43.0" 2024-03-29T05:13:48.980 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:48 smithi181 ceph-mon[106466]: pgmap v1297: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:48.980 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:48 smithi181 ceph-mon[101970]: pgmap v1297: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:49.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:48 smithi187 ceph-mon[93536]: pgmap v1297: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:50.121 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:13:50.121 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:13:50.753 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:50 smithi181 ceph-mon[106466]: pgmap v1298: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:50.753 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:50 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1200024578' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:50.753 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:50 smithi181 ceph-mon[101970]: pgmap v1298: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:50.754 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:50 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1200024578' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:51.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:50 smithi187 ceph-mon[93536]: pgmap v1298: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:51.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:50 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1200024578' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:53.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:52 smithi187 ceph-mon[93536]: pgmap v1299: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:53.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:52 smithi181 ceph-mon[106466]: pgmap v1299: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:53.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:52 smithi181 ceph-mon[101970]: pgmap v1299: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:53.778 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:13:54.046 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:13:54.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:54 smithi181 ceph-mon[101970]: pgmap v1300: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:54.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:54 smithi181 ceph-mon[106466]: pgmap v1300: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:55.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:54 smithi187 ceph-mon[93536]: pgmap v1300: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:56.092 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:13:56.093 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:13:56.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:56 smithi181 ceph-mon[106466]: pgmap v1301: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:56.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:56 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2614187635' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:56.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:13:56.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:56 smithi181 ceph-mon[101970]: pgmap v1301: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:56.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:56 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2614187635' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:56.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:13:57.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:56 smithi187 ceph-mon[93536]: pgmap v1301: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:57.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:56 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2614187635' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:13:57.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:13:59.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:13:58 smithi187 ceph-mon[93536]: pgmap v1302: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:59.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:13:58 smithi181 ceph-mon[101970]: pgmap v1302: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:59.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:13:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:13:58] "GET /metrics HTTP/1.1" 200 34117 "" "Prometheus/2.43.0" 2024-03-29T05:13:59.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:13:58 smithi181 ceph-mon[106466]: pgmap v1302: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:13:59.727 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:13:59.992 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:14:01.032 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:00 smithi181 ceph-mon[106466]: pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:01.032 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:00 smithi181 ceph-mon[101970]: pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:01.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:00 smithi187 ceph-mon[93536]: pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:01.803 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:14:01.803 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:14:02.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:02 smithi181 ceph-mon[106466]: pgmap v1304: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:02.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:02 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/4086662702' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:02.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:02 smithi181 ceph-mon[101970]: pgmap v1304: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:02.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:02 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/4086662702' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:03.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:02 smithi187 ceph-mon[93536]: pgmap v1304: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:03.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:02 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/4086662702' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:05.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:04 smithi187 ceph-mon[93536]: pgmap v1305: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:05.202 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:04 smithi181 ceph-mon[106466]: pgmap v1305: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:05.202 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:04 smithi181 ceph-mon[101970]: pgmap v1305: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:05.522 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:14:05.793 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:14:06.912 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:06 smithi181 ceph-mon[106466]: pgmap v1306: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:06.913 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:06 smithi181 ceph-mon[101970]: pgmap v1306: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:07.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:06 smithi187 ceph-mon[93536]: pgmap v1306: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:07.660 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:14:07.660 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:14:08.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:07 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1367343772' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:08.186 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:07 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1367343772' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:08.186 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:07 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1367343772' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:09.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:08 smithi187 ceph-mon[93536]: pgmap v1307: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:09.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:08 smithi181 ceph-mon[101970]: pgmap v1307: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:09.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:08 smithi181 ceph-mon[106466]: pgmap v1307: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:09.202 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:14:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:14:08] "GET /metrics HTTP/1.1" 200 34117 "" "Prometheus/2.43.0" 2024-03-29T05:14:11.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:10 smithi187 ceph-mon[93536]: pgmap v1308: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:11.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:10 smithi181 ceph-mon[101970]: pgmap v1308: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:11.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:10 smithi181 ceph-mon[106466]: pgmap v1308: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:11.214 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:14:11.486 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:14:11.940 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:14:11.940 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:14:12.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:14:12.995 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:12 smithi181 ceph-mon[106466]: pgmap v1309: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:12.995 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:12 smithi181 ceph-mon[101970]: pgmap v1309: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:13.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:12 smithi187 ceph-mon[93536]: pgmap v1309: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:13.498 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:14:13.498 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:14:14.051 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:13 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/940179165' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:14.051 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:13 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/940179165' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:14.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:13 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/940179165' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:15.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:14 smithi187 ceph-mon[93536]: pgmap v1310: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:15.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:14 smithi181 ceph-mon[101970]: pgmap v1310: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:15.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:14 smithi181 ceph-mon[106466]: pgmap v1310: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:17.077 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:14:17.108 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:16 smithi181 ceph-mon[101970]: pgmap v1311: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:17.108 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:16 smithi181 ceph-mon[106466]: pgmap v1311: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:17.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:16 smithi187 ceph-mon[93536]: pgmap v1311: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:17.343 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:14:18.986 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:18 smithi181 ceph-mon[101970]: pgmap v1312: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:18.986 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:14:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:14:18] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2024-03-29T05:14:18.987 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:18 smithi181 ceph-mon[106466]: pgmap v1312: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:19.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:18 smithi187 ceph-mon[93536]: pgmap v1312: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:19.224 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:14:19.224 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:14:19.870 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:19 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2449886738' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:19.870 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:19 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2449886738' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:20.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:19 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2449886738' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:21.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:20 smithi187 ceph-mon[93536]: pgmap v1313: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:21.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:20 smithi181 ceph-mon[101970]: pgmap v1313: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:21.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:20 smithi181 ceph-mon[106466]: pgmap v1313: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:22.900 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:14:23.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:22 smithi187 ceph-mon[93536]: pgmap v1314: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:23.164 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:14:23.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:22 smithi181 ceph-mon[101970]: pgmap v1314: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:23.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:22 smithi181 ceph-mon[106466]: pgmap v1314: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:24.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:24 smithi181 ceph-mon[101970]: pgmap v1315: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:24.952 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:24 smithi181 ceph-mon[106466]: pgmap v1315: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:25.027 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:14:25.027 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:14:25.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:24 smithi187 ceph-mon[93536]: pgmap v1315: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:25.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:25 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3806643723' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:25.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:25 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3806643723' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:26.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:25 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3806643723' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:27.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:26 smithi187 ceph-mon[93536]: pgmap v1316: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:27.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:14:27.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:26 smithi181 ceph-mon[106466]: pgmap v1316: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:27.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:14:27.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:26 smithi181 ceph-mon[101970]: pgmap v1316: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:27.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:14:28.648 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:14:28.914 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:14:28.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:28 smithi181 ceph-mon[106466]: pgmap v1317: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:28.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:28 smithi181 ceph-mon[101970]: pgmap v1317: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:29.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:28 smithi187 ceph-mon[93536]: pgmap v1317: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:29.205 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:14:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:14:28] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.43.0" 2024-03-29T05:14:30.923 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:14:30.923 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:14:31.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:30 smithi187 ceph-mon[93536]: pgmap v1318: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:31.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:30 smithi181 ceph-mon[106466]: pgmap v1318: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:31.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:30 smithi181 ceph-mon[101970]: pgmap v1318: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:32.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:31 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/942641024' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:32.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:31 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/942641024' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:32.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:31 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/942641024' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:33.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:32 smithi187 ceph-mon[93536]: pgmap v1319: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:33.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:32 smithi181 ceph-mon[101970]: pgmap v1319: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:33.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:32 smithi181 ceph-mon[106466]: pgmap v1319: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:34.547 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:14:34.813 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:14:34.935 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:34 smithi181 ceph-mon[101970]: pgmap v1320: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:34.935 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:34 smithi181 ceph-mon[106466]: pgmap v1320: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:35.132 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:34 smithi187 ceph-mon[93536]: pgmap v1320: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:36.785 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:14:36.785 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:14:37.100 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:36 smithi181 ceph-mon[101970]: pgmap v1321: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:37.100 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:36 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3522219319' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:37.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:36 smithi181 ceph-mon[106466]: pgmap v1321: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:37.100 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:36 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3522219319' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:37.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:36 smithi187 ceph-mon[93536]: pgmap v1321: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:37.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:36 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3522219319' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:39.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:38 smithi187 ceph-mon[93536]: pgmap v1322: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:39.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:38 smithi181 ceph-mon[101970]: pgmap v1322: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:39.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:38 smithi181 ceph-mon[106466]: pgmap v1322: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:39.201 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:14:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:14:38] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.43.0" 2024-03-29T05:14:40.435 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:14:40.690 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:14:41.103 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:40 smithi181 ceph-mon[101970]: pgmap v1323: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:41.104 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:40 smithi181 ceph-mon[106466]: pgmap v1323: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:41.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:40 smithi187 ceph-mon[93536]: pgmap v1323: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:42.102 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:14:42.103 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:14:42.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:14:42.624 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:14:42.624 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:14:43.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:42 smithi187 ceph-mon[93536]: pgmap v1324: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:43.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:42 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1804213118' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:43.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:42 smithi181 ceph-mon[106466]: pgmap v1324: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:43.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:42 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1804213118' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:43.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:42 smithi181 ceph-mon[101970]: pgmap v1324: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:43.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:42 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1804213118' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:44.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:43 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:14:44.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:43 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:14:44.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:43 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:14:45.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:44 smithi187 ceph-mon[93536]: pgmap v1325: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:45.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:44 smithi181 ceph-mon[101970]: pgmap v1325: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:45.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:44 smithi181 ceph-mon[106466]: pgmap v1325: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:46.348 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:14:46.745 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:14:47.079 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:46 smithi181 ceph-mon[106466]: pgmap v1326: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:47.079 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:46 smithi181 ceph-mon[101970]: pgmap v1326: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:47.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:46 smithi187 ceph-mon[93536]: pgmap v1326: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:48.856 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-mon[106466]: pgmap v1327: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:48.857 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:14:48.857 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:14:48.857 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:14:48.857 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:14:48.857 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:14:48.857 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:14:48.857 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:14:48.857 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-mon[101970]: pgmap v1327: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:48.857 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:14:48.857 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:14:48.858 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:14:48.858 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:14:48.858 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:14:48.858 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:14:48.858 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:14:49.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:48 smithi187 ceph-mon[93536]: pgmap v1327: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:49.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:48 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:14:49.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:48 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:14:49.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:48 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:14:49.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:48 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:14:49.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:48 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:14:49.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:48 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:14:49.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:48 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:14:49.201 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:14:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:14:48] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.43.0" 2024-03-29T05:14:49.329 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:14:49.329 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:14:49.903 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:49 smithi181 ceph-mon[106466]: pgmap v1328: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:49.903 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:49 smithi181 ceph-mon[106466]: pgmap v1329: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:49.903 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:49 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2246284319' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:49.904 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:49 smithi181 ceph-mon[101970]: pgmap v1328: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:49.904 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:49 smithi181 ceph-mon[101970]: pgmap v1329: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:49.904 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:49 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2246284319' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:50.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:49 smithi187 ceph-mon[93536]: pgmap v1328: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:50.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:49 smithi187 ceph-mon[93536]: pgmap v1329: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:50.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:49 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2246284319' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:52.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:51 smithi187 ceph-mon[93536]: pgmap v1330: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:52.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:51 smithi181 ceph-mon[106466]: pgmap v1330: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:52.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:51 smithi181 ceph-mon[101970]: pgmap v1330: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:52.989 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:14:53.257 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:14:53.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:53 smithi181 ceph-mon[106466]: pgmap v1331: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:53.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:53 smithi181 ceph-mon[101970]: pgmap v1331: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:54.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:53 smithi187 ceph-mon[93536]: pgmap v1331: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:55.125 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:14:55.125 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:14:56.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:55 smithi187 ceph-mon[93536]: pgmap v1332: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:56.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:55 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/122864916' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:56.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:55 smithi181 ceph-mon[101970]: pgmap v1332: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:56.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:55 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/122864916' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:56.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:55 smithi181 ceph-mon[106466]: pgmap v1332: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:56.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:55 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/122864916' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:14:57.133 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:14:57.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:14:57.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:14:58.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:57 smithi187 ceph-mon[93536]: pgmap v1333: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:58.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:57 smithi181 ceph-mon[106466]: pgmap v1333: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:58.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:57 smithi181 ceph-mon[101970]: pgmap v1333: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:14:58.735 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:14:59.005 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:14:59.127 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:14:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:14:58] "GET /metrics HTTP/1.1" 200 34108 "" "Prometheus/2.43.0" 2024-03-29T05:15:00.091 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:14:59 smithi181 ceph-mon[106466]: pgmap v1334: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:00.092 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:14:59 smithi181 ceph-mon[101970]: pgmap v1334: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:00.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:14:59 smithi187 ceph-mon[93536]: pgmap v1334: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:00.854 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:15:00.854 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:15:02.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:01 smithi187 ceph-mon[93536]: pgmap v1335: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:02.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:01 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2823740991' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:02.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:01 smithi181 ceph-mon[101970]: pgmap v1335: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:02.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:01 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2823740991' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:02.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:01 smithi181 ceph-mon[106466]: pgmap v1335: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:02.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:01 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2823740991' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:04.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:03 smithi187 ceph-mon[93536]: pgmap v1336: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:04.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:03 smithi181 ceph-mon[106466]: pgmap v1336: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:04.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:03 smithi181 ceph-mon[101970]: pgmap v1336: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:04.500 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:15:04.765 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:15:05.906 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:05 smithi181 ceph-mon[101970]: pgmap v1337: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:05.906 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:05 smithi181 ceph-mon[106466]: pgmap v1337: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:06.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:05 smithi187 ceph-mon[93536]: pgmap v1337: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:06.765 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:15:06.765 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:15:07.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:06 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2994834661' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:07.138 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:06 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2994834661' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:07.138 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:06 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2994834661' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:08.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:07 smithi187 ceph-mon[93536]: pgmap v1338: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:08.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:07 smithi181 ceph-mon[106466]: pgmap v1338: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:08.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:07 smithi181 ceph-mon[101970]: pgmap v1338: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:09.373 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:15:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:15:08] "GET /metrics HTTP/1.1" 200 34108 "" "Prometheus/2.43.0" 2024-03-29T05:15:10.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:09 smithi187 ceph-mon[93536]: pgmap v1339: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:10.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:09 smithi181 ceph-mon[101970]: pgmap v1339: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:10.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:09 smithi181 ceph-mon[106466]: pgmap v1339: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:10.459 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:15:10.715 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:15:11.929 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:11 smithi181 ceph-mon[101970]: pgmap v1340: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:11.929 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:15:11.929 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:11 smithi181 ceph-mon[106466]: pgmap v1340: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:11.929 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:15:12.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:11 smithi187 ceph-mon[93536]: pgmap v1340: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:12.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:15:12.711 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:15:12.712 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:15:13.010 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:12 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2923601509' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:13.011 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:12 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2923601509' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:13.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:12 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2923601509' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:14.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:13 smithi187 ceph-mon[93536]: pgmap v1341: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:14.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:13 smithi181 ceph-mon[101970]: pgmap v1341: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:14.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:13 smithi181 ceph-mon[106466]: pgmap v1341: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:16.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:15 smithi187 ceph-mon[93536]: pgmap v1342: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:16.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:15 smithi181 ceph-mon[101970]: pgmap v1342: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:16.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:15 smithi181 ceph-mon[106466]: pgmap v1342: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:16.327 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:15:16.595 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:15:17.989 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:17 smithi181 ceph-mon[101970]: pgmap v1343: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:17.989 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:17 smithi181 ceph-mon[106466]: pgmap v1343: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:18.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:17 smithi187 ceph-mon[93536]: pgmap v1343: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:18.466 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:15:18.466 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:15:19.108 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:15:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:15:18] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.43.0" 2024-03-29T05:15:19.108 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:18 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/179874129' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:19.109 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:18 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/179874129' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:19.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:18 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/179874129' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:20.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:19 smithi187 ceph-mon[93536]: pgmap v1344: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:20.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:19 smithi181 ceph-mon[101970]: pgmap v1344: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:20.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:19 smithi181 ceph-mon[106466]: pgmap v1344: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:22.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:21 smithi187 ceph-mon[93536]: pgmap v1345: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:22.134 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:15:22.165 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:21 smithi181 ceph-mon[101970]: pgmap v1345: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:22.165 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:21 smithi181 ceph-mon[106466]: pgmap v1345: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:22.402 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:15:24.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:23 smithi187 ceph-mon[93536]: pgmap v1346: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:24.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:23 smithi181 ceph-mon[101970]: pgmap v1346: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:24.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:23 smithi181 ceph-mon[106466]: pgmap v1346: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:24.304 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:15:24.304 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:15:24.879 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:24 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3375191637' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:24.880 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:24 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3375191637' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:25.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:24 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3375191637' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:26.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:25 smithi187 ceph-mon[93536]: pgmap v1347: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:26.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:25 smithi181 ceph-mon[101970]: pgmap v1347: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:26.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:25 smithi181 ceph-mon[106466]: pgmap v1347: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:27.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:26 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:15:27.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:26 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:15:27.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:26 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:15:27.905 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:15:28.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:27 smithi187 ceph-mon[93536]: pgmap v1348: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:28.170 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:15:28.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:27 smithi181 ceph-mon[101970]: pgmap v1348: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:28.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:27 smithi181 ceph-mon[106466]: pgmap v1348: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:29.201 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:15:28 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:15:28] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2024-03-29T05:15:29.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:29 smithi181 ceph-mon[106466]: pgmap v1349: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:29.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:29 smithi181 ceph-mon[101970]: pgmap v1349: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:30.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:29 smithi187 ceph-mon[93536]: pgmap v1349: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:30.145 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:15:30.145 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:15:31.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:30 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2156001420' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:31.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:30 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2156001420' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:31.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:30 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2156001420' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:32.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:31 smithi187 ceph-mon[93536]: pgmap v1350: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:32.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:31 smithi181 ceph-mon[106466]: pgmap v1350: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:32.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:31 smithi181 ceph-mon[101970]: pgmap v1350: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:33.863 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:15:34.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:33 smithi187 ceph-mon[93536]: pgmap v1351: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:34.131 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:15:34.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:33 smithi181 ceph-mon[106466]: pgmap v1351: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:34.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:33 smithi181 ceph-mon[101970]: pgmap v1351: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:35.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:35 smithi181 ceph-mon[101970]: pgmap v1352: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:35.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:35 smithi181 ceph-mon[106466]: pgmap v1352: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:36.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:35 smithi187 ceph-mon[93536]: pgmap v1352: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:36.136 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:15:36.137 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:15:37.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:36 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2364422054' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:37.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:36 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2364422054' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:37.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:36 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2364422054' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:38.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:37 smithi187 ceph-mon[93536]: pgmap v1353: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:38.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:37 smithi181 ceph-mon[106466]: pgmap v1353: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:38.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:37 smithi181 ceph-mon[101970]: pgmap v1353: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:39.450 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:15:38 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:15:38] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2024-03-29T05:15:39.947 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:15:40.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:39 smithi181 ceph-mon[106466]: pgmap v1354: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:40.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:39 smithi181 ceph-mon[101970]: pgmap v1354: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:40.225 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:15:40.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:39 smithi187 ceph-mon[93536]: pgmap v1354: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:42.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:41 smithi181 ceph-mon[101970]: pgmap v1355: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:42.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:41 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:15:42.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:41 smithi181 ceph-mon[106466]: pgmap v1355: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:42.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:41 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:15:42.233 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:15:42.233 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:15:42.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:41 smithi187 ceph-mon[93536]: pgmap v1355: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:42.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:41 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:15:43.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:42 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/818748803' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:43.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:42 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/818748803' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:43.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:42 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/818748803' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:44.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:43 smithi181 ceph-mon[101970]: pgmap v1356: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:44.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:43 smithi181 ceph-mon[106466]: pgmap v1356: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:44.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:43 smithi187 ceph-mon[93536]: pgmap v1356: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:45.918 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:15:46.185 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:15:46.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:45 smithi181 ceph-mon[101970]: pgmap v1357: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:46.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:45 smithi181 ceph-mon[106466]: pgmap v1357: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:46.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:45 smithi187 ceph-mon[93536]: pgmap v1357: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:48.002 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:47 smithi181 ceph-mon[106466]: pgmap v1358: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:48.002 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:47 smithi181 ceph-mon[101970]: pgmap v1358: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:48.243 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:15:48.244 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:15:48.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:47 smithi187 ceph-mon[93536]: pgmap v1358: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:49.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:48 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/775025205' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:49.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:48 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:15:49.201 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:15:48 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:15:48] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.43.0" 2024-03-29T05:15:49.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:48 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/775025205' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:49.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:48 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:15:49.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:48 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/775025205' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:49.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:48 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T05:15:50.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:49 smithi181 ceph-mon[106466]: pgmap v1359: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:50.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:49 smithi181 ceph-mon[101970]: pgmap v1359: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:50.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:49 smithi187 ceph-mon[93536]: pgmap v1359: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:51.949 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:15:51.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:51 smithi181 ceph-mon[101970]: pgmap v1360: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:51.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:51 smithi181 ceph-mon[106466]: pgmap v1360: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:52.292 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:51 smithi187 ceph-mon[93536]: pgmap v1360: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:52.673 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:15:53.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:53 smithi181 ceph-mon[106466]: pgmap v1361: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:53.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:53 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:15:53.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:53 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:15:53.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:53 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:15:53.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:53 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:15:53.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:53 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:15:53.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:53 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:15:53.951 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:53 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:15:53.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:53 smithi181 ceph-mon[101970]: pgmap v1361: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:53.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:53 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:15:53.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:53 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:15:53.951 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:53 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:15:53.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:53 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:15:53.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:53 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:15:53.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:53 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:15:53.952 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:53 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:15:54.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:53 smithi187 ceph-mon[93536]: pgmap v1361: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:54.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:53 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:15:54.130 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:53 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:15:54.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:53 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:15:54.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:53 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:15:54.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:53 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T05:15:54.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:53 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T05:15:54.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:53 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' 2024-03-29T05:15:54.787 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:15:54.787 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:15:55.154 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:54 smithi181 ceph-mon[106466]: pgmap v1362: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:55.154 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:54 smithi181 ceph-mon[106466]: pgmap v1363: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:55.154 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:54 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2037585507' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:55.154 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:54 smithi181 ceph-mon[101970]: pgmap v1362: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:55.154 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:54 smithi181 ceph-mon[101970]: pgmap v1363: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:55.155 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:54 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2037585507' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:55.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:54 smithi187 ceph-mon[93536]: pgmap v1362: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:55.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:54 smithi187 ceph-mon[93536]: pgmap v1363: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:55.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:54 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2037585507' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:15:57.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:56 smithi181 ceph-mon[101970]: pgmap v1364: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:57.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:56 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:15:57.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:56 smithi181 ceph-mon[106466]: pgmap v1364: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:57.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:56 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:15:57.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:56 smithi187 ceph-mon[93536]: pgmap v1364: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:57.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:56 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:15:58.548 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:15:58.814 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:15:59.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:15:58 smithi181 ceph-mon[106466]: pgmap v1365: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:59.201 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:15:58 smithi181 ceph-mon[101970]: pgmap v1365: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:15:59.201 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:15:58 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:15:58] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-03-29T05:15:59.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:15:58 smithi187 ceph-mon[93536]: pgmap v1365: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:00.744 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:16:00.744 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:16:01.101 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:00 smithi181 ceph-mon[101970]: pgmap v1366: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:01.102 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:00 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/3654226235' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:16:01.102 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:00 smithi181 ceph-mon[106466]: pgmap v1366: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:01.102 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:00 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/3654226235' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:16:01.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:00 smithi187 ceph-mon[93536]: pgmap v1366: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:01.381 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:00 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/3654226235' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:16:03.183 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:02 smithi187 ceph-mon[93536]: pgmap v1367: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:03.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:02 smithi181 ceph-mon[106466]: pgmap v1367: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:03.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:02 smithi181 ceph-mon[101970]: pgmap v1367: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:04.479 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:16:04.748 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:16:05.147 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:04 smithi181 ceph-mon[106466]: pgmap v1368: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:05.147 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:04 smithi181 ceph-mon[101970]: pgmap v1368: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:05.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:04 smithi187 ceph-mon[93536]: pgmap v1368: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:06.812 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:16:06.813 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:16:07.108 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:06 smithi181 ceph-mon[106466]: pgmap v1369: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:07.108 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:06 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/2612553474' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:16:07.108 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:06 smithi181 ceph-mon[101970]: pgmap v1369: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:07.108 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:06 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/2612553474' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:16:07.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:06 smithi187 ceph-mon[93536]: pgmap v1369: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:07.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:06 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/2612553474' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:16:09.200 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:08 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:16:08] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-03-29T05:16:09.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:08 smithi181 ceph-mon[101970]: pgmap v1370: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:09.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:08 smithi181 ceph-mon[106466]: pgmap v1370: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:09.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:08 smithi187 ceph-mon[93536]: pgmap v1370: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:10.539 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:16:10.793 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:16:11.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:10 smithi181 ceph-mon[106466]: pgmap v1371: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:11.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:10 smithi181 ceph-mon[101970]: pgmap v1371: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:11.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:10 smithi187 ceph-mon[93536]: pgmap v1371: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:11.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:11 smithi181 ceph-mon[106466]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:16:11.950 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:11 smithi181 ceph-mon[101970]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:16:12.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:11 smithi187 ceph-mon[93536]: from='mgr.25337 172.21.15.181:0/2140399984' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T05:16:12.842 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:16:12.842 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:16:13.165 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:12 smithi181 ceph-mon[106466]: pgmap v1372: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:13.165 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:12 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1471039642' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:16:13.165 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:12 smithi181 ceph-mon[101970]: pgmap v1372: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:13.165 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:12 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1471039642' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:16:13.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:12 smithi187 ceph-mon[93536]: pgmap v1372: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:13.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:12 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1471039642' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:16:15.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:14 smithi181 ceph-mon[101970]: pgmap v1373: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:15.200 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:14 smithi181 ceph-mon[106466]: pgmap v1373: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:15.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:14 smithi187 ceph-mon[93536]: pgmap v1373: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:16.550 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T05:16:16.816 INFO:teuthology.orchestra.run.smithi181.stderr:Inferring config /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T05:16:17.200 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:16 smithi181 ceph-mon[101970]: pgmap v1374: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:17.201 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:16 smithi181 ceph-mon[106466]: pgmap v1374: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:17.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:16 smithi187 ceph-mon[93536]: pgmap v1374: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:18.891 INFO:teuthology.orchestra.run.smithi181.stdout: 2024-03-29T05:16:18.891 INFO:teuthology.orchestra.run.smithi181.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T05:16:19.197 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:18 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: ::ffff:172.21.15.187 - - [29/Mar/2024:05:16:18] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.43.0" 2024-03-29T05:16:19.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:18 smithi181 ceph-mon[106466]: pgmap v1375: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:19.197 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:18 smithi181 ceph-mon[106466]: from='client.? 172.21.15.181:0/1270505155' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:16:19.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:18 smithi181 ceph-mon[101970]: pgmap v1375: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:19.197 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:18 smithi181 ceph-mon[101970]: from='client.? 172.21.15.181:0/1270505155' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:16:19.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:18 smithi187 ceph-mon[93536]: pgmap v1375: 73 pgs: 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-03-29T05:16:19.380 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:18 smithi187 ceph-mon[93536]: from='client.? 172.21.15.181:0/1270505155' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T05:16:19.497 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-03-29T05:16:19.740 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=6ef80694eb64479cb4f9f341f9fbf018 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-03-29T05:16:19.742 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-03-29T05:16:19.751 INFO:tasks.cephadm:Teardown begin 2024-03-29T05:16:19.752 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-29T05:16:19.786 DEBUG:teuthology.orchestra.run.smithi187:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-29T05:16:19.821 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-03-29T05:16:19.822 DEBUG:teuthology.orchestra.run.smithi181:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-29T05:16:19.841 DEBUG:teuthology.orchestra.run.smithi187:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-29T05:16:19.879 INFO:tasks.cephadm:Stopping all daemons... 2024-03-29T05:16:19.879 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-03-29T05:16:19.879 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.a 2024-03-29T05:16:20.178 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:19 smithi181 systemd[1]: Stopping Ceph mon.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:16:20.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a[101947]: 2024-03-29T05:16:20.173+0000 7fb7daea9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T05:16:20.450 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a[101947]: 2024-03-29T05:16:20.173+0000 7fb7daea9700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-03-29T05:16:20.451 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:20 smithi181 podman[162354]: 2024-03-29 05:16:20.327076352 +0000 UTC m=+0.264941329 container died 4e5d3325d991c36d51de6dea6c47c4695bb2443f666e6b95e63cfe05ab4e4d94 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-03-29T05:16:20.451 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:16:20] ENGINE Bus STOPPING 2024-03-29T05:16:20.451 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:16:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T05:16:20.451 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:16:20] ENGINE Bus STOPPED 2024-03-29T05:16:20.451 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:16:20] ENGINE Bus STARTING 2024-03-29T05:16:20.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:16:20] ENGINE Serving on http://:::9283 2024-03-29T05:16:20.452 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:16:20] ENGINE Bus STARTED 2024-03-29T05:16:20.758 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:20 smithi181 podman[162354]: 2024-03-29 05:16:20.738782877 +0000 UTC m=+0.676647862 container cleanup 4e5d3325d991c36d51de6dea6c47c4695bb2443f666e6b95e63cfe05ab4e4d94 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-03-29T05:16:20.758 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:20 smithi181 bash[162354]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a 2024-03-29T05:16:21.214 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.a.service' 2024-03-29T05:16:21.238 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:20 smithi181 podman[162450]: 2024-03-29 05:16:20.965242316 +0000 UTC m=+0.639249300 container remove 4e5d3325d991c36d51de6dea6c47c4695bb2443f666e6b95e63cfe05ab4e4d94 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-a, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, GIT_BRANCH=HEAD) 2024-03-29T05:16:21.238 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:21 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.a.service: Deactivated successfully. 2024-03-29T05:16:21.238 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:21 smithi181 systemd[1]: Stopped Ceph mon.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T05:16:21.239 INFO:journalctl@ceph.mon.a.smithi181.stdout:Mar 29 05:16:21 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.a.service: Consumed 30.034s CPU time. 2024-03-29T05:16:21.292 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T05:16:21.292 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-03-29T05:16:21.292 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-03-29T05:16:21.292 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.c 2024-03-29T05:16:21.650 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:21 smithi181 systemd[1]: Stopping Ceph mon.c for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:16:21.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[106443]: 2024-03-29T05:16:21.645+0000 7f0187721700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T05:16:21.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c[106443]: 2024-03-29T05:16:21.645+0000 7f0187721700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-03-29T05:16:21.950 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:21 smithi181 podman[162584]: 2024-03-29 05:16:21.717932211 +0000 UTC m=+0.206836301 container died 10f02d533230221001aa56d0e2d12679381982e09fd7b5fa62741baad2c211d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, ceph=True) 2024-03-29T05:16:21.951 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:16:21] ENGINE Bus STOPPING 2024-03-29T05:16:21.951 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:16:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T05:16:21.951 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:16:21] ENGINE Bus STOPPED 2024-03-29T05:16:21.951 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:16:21] ENGINE Bus STARTING 2024-03-29T05:16:21.951 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:16:21] ENGINE Serving on http://:::9283 2024-03-29T05:16:21.951 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:21 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: [29/Mar/2024:05:16:21] ENGINE Bus STARTED 2024-03-29T05:16:22.583 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:22 smithi181 podman[162584]: 2024-03-29 05:16:22.377639474 +0000 UTC m=+0.866543598 container cleanup 10f02d533230221001aa56d0e2d12679381982e09fd7b5fa62741baad2c211d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-03-29T05:16:22.583 INFO:journalctl@ceph.mon.c.smithi181.stdout:Mar 29 05:16:22 smithi181 bash[162584]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-c 2024-03-29T05:16:22.801 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.c.service' 2024-03-29T05:16:22.839 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T05:16:22.839 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-03-29T05:16:22.839 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-03-29T05:16:22.839 DEBUG:teuthology.orchestra.run.smithi187:> sudo systemctl stop ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.b 2024-03-29T05:16:23.131 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:22 smithi187 systemd[1]: Stopping Ceph mon.b for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:16:23.630 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:23 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[93512]: 2024-03-29T05:16:23.159+0000 7f767d623700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T05:16:23.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:23 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b[93512]: 2024-03-29T05:16:23.159+0000 7f767d623700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-03-29T05:16:23.631 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:23 smithi187 podman[114254]: 2024-03-29 05:16:23.354259043 +0000 UTC m=+0.321675054 container died 4de0ae3c44c36f2dc00ac3f31d42bb83b5867282720d6b8baf7aa7fb7949eea5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T05:16:24.096 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:23 smithi187 podman[114254]: 2024-03-29 05:16:23.865878443 +0000 UTC m=+0.833294449 container cleanup 4de0ae3c44c36f2dc00ac3f31d42bb83b5867282720d6b8baf7aa7fb7949eea5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362) 2024-03-29T05:16:24.096 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:23 smithi187 bash[114254]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b 2024-03-29T05:16:24.336 DEBUG:teuthology.orchestra.run.smithi187:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.b.service' 2024-03-29T05:16:24.359 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:24 smithi187 podman[114273]: 2024-03-29 05:16:24.092435467 +0000 UTC m=+0.739753597 container remove 4de0ae3c44c36f2dc00ac3f31d42bb83b5867282720d6b8baf7aa7fb7949eea5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-03-29T05:16:24.360 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:24 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.b.service: Deactivated successfully. 2024-03-29T05:16:24.360 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:24 smithi187 systemd[1]: Stopped Ceph mon.b for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T05:16:24.360 INFO:journalctl@ceph.mon.b.smithi187.stdout:Mar 29 05:16:24 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mon.b.service: Consumed 13.791s CPU time. 2024-03-29T05:16:24.382 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T05:16:24.382 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-03-29T05:16:24.382 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-03-29T05:16:24.382 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.y 2024-03-29T05:16:24.680 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:24 smithi181 systemd[1]: Stopping Ceph mgr.y for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:16:24.950 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:24 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T05:16:24.675+0000 7f67b3c0c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T05:16:24.950 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:24 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y[92269]: 2024-03-29T05:16:24.675+0000 7f67b3c0c700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-03-29T05:16:24.950 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:24 smithi181 podman[162726]: 2024-03-29 05:16:24.70761507 +0000 UTC m=+0.135866026 container died b07d84f4244a64d51076ac149797acee0011f1bb9f7faf2f8d6e0ad95ccb2005 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240326, RELEASE=squid-8da2362) 2024-03-29T05:16:25.446 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:25 smithi181 podman[162726]: 2024-03-29 05:16:25.165844849 +0000 UTC m=+0.594095847 container cleanup b07d84f4244a64d51076ac149797acee0011f1bb9f7faf2f8d6e0ad95ccb2005 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T05:16:25.446 INFO:journalctl@ceph.mgr.y.smithi181.stdout:Mar 29 05:16:25 smithi181 bash[162726]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-y 2024-03-29T05:16:25.590 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.y.service' 2024-03-29T05:16:25.627 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T05:16:25.628 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-03-29T05:16:25.628 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-03-29T05:16:25.628 DEBUG:teuthology.orchestra.run.smithi187:> sudo systemctl stop ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.x 2024-03-29T05:16:25.983 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 05:16:25 smithi187 systemd[1]: Stopping Ceph mgr.x for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:16:26.250 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 05:16:25 smithi187 podman[114385]: 2024-03-29 05:16:25.97907828 +0000 UTC m=+0.158121592 container died 19bc315720bdf7f7413bb32a7c1da0665e3c61c8b6854f9b33ad5c64f3638fbc (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362) 2024-03-29T05:16:26.600 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 05:16:26 smithi187 podman[114385]: 2024-03-29 05:16:26.336679191 +0000 UTC m=+0.515722533 container cleanup 19bc315720bdf7f7413bb32a7c1da0665e3c61c8b6854f9b33ad5c64f3638fbc (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-8da2362) 2024-03-29T05:16:26.600 INFO:journalctl@ceph.mgr.x.smithi187.stdout:Mar 29 05:16:26 smithi187 bash[114385]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-mgr-x 2024-03-29T05:16:26.761 DEBUG:teuthology.orchestra.run.smithi187:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@mgr.x.service' 2024-03-29T05:16:26.797 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T05:16:26.798 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-03-29T05:16:26.798 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-03-29T05:16:26.798 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.0 2024-03-29T05:16:27.200 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 05:16:26 smithi181 systemd[1]: Stopping Ceph osd.0 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:16:27.700 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 05:16:27 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0[115096]: 2024-03-29T05:16:27.264+0000 7ffaf2dad700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T05:16:27.700 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 05:16:27 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0[115096]: 2024-03-29T05:16:27.264+0000 7ffaf2dad700 -1 osd.0 972 *** Got signal Terminated *** 2024-03-29T05:16:27.700 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 05:16:27 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0[115096]: 2024-03-29T05:16:27.264+0000 7ffaf2dad700 -1 osd.0 972 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T05:16:30.450 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:16:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:16:30.159Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:16:30.450 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:16:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:16:30.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:16:32.942 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 05:16:32 smithi181 podman[162855]: 2024-03-29 05:16:32.636520394 +0000 UTC m=+5.653955720 container died 03d36b79ef26d0e5c3783209243761f73aed0cc0e7999558db8183342e0adb87 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, org.label-schema.build-date=20240326) 2024-03-29T05:16:33.200 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 05:16:33 smithi181 podman[162855]: 2024-03-29 05:16:33.112002505 +0000 UTC m=+6.129437821 container cleanup 03d36b79ef26d0e5c3783209243761f73aed0cc0e7999558db8183342e0adb87 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, ceph=True, io.buildah.version=1.33.5) 2024-03-29T05:16:33.200 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 05:16:33 smithi181 bash[162855]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0 2024-03-29T05:16:33.700 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 05:16:33 smithi181 podman[162877]: 2024-03-29 05:16:33.288632974 +0000 UTC m=+0.651544787 container remove 03d36b79ef26d0e5c3783209243761f73aed0cc0e7999558db8183342e0adb87 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T05:16:34.037 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 05:16:33 smithi181 podman[162972]: 2024-03-29 05:16:33.623665355 +0000 UTC m=+0.017977111 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:16:34.037 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 05:16:33 smithi181 podman[162972]: 2024-03-29 05:16:33.73100132 +0000 UTC m=+0.125313070 container create c1b67b9459ab181578f75dc530a251d58ee528c682a96eeb632ebd8975bd0960 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-deactivate, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-03-29T05:16:34.037 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 05:16:33 smithi181 podman[162972]: 2024-03-29 05:16:33.94158869 +0000 UTC m=+0.335900439 container init c1b67b9459ab181578f75dc530a251d58ee528c682a96eeb632ebd8975bd0960 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-03-29T05:16:34.037 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 05:16:33 smithi181 podman[162972]: 2024-03-29 05:16:33.947257606 +0000 UTC m=+0.341569355 container start c1b67b9459ab181578f75dc530a251d58ee528c682a96eeb632ebd8975bd0960 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240326) 2024-03-29T05:16:34.037 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 05:16:34 smithi181 podman[162972]: 2024-03-29 05:16:34.033091338 +0000 UTC m=+0.427403100 container attach c1b67b9459ab181578f75dc530a251d58ee528c682a96eeb632ebd8975bd0960 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T05:16:34.313 INFO:journalctl@ceph.osd.0.smithi181.stdout:Mar 29 05:16:34 smithi181 podman[162972]: 2024-03-29 05:16:34.248353988 +0000 UTC m=+0.642665730 container died c1b67b9459ab181578f75dc530a251d58ee528c682a96eeb632ebd8975bd0960 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-0-deactivate, RELEASE=squid-8da2362, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-03-29T05:16:34.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:16:34 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:16:34.309Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:16:34.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:16:34 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:16:34.310Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:16:34.946 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.0.service' 2024-03-29T05:16:34.988 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T05:16:34.988 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-03-29T05:16:34.989 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-03-29T05:16:34.989 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.1 2024-03-29T05:16:35.200 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:16:35 smithi181 systemd[1]: Stopping Ceph osd.1 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:16:35.700 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:16:35 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1[121432]: 2024-03-29T05:16:35.400+0000 7f6196b70700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T05:16:35.700 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:16:35 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1[121432]: 2024-03-29T05:16:35.400+0000 7f6196b70700 -1 osd.1 972 *** Got signal Terminated *** 2024-03-29T05:16:35.700 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:16:35 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1[121432]: 2024-03-29T05:16:35.400+0000 7f6196b70700 -1 osd.1 972 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T05:16:40.450 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:16:40 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:16:40.159Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:16:40.451 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:16:40 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:16:40.159Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:16:40.451 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:16:40 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:16:40.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:16:41.045 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:16:40 smithi181 podman[163107]: 2024-03-29 05:16:40.767156407 +0000 UTC m=+5.491732388 container died cb0b10fc92dd2dcc778a1b81e1b7bd9da258c43e906f1c84a7c9ebd72d7c2040 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T05:16:41.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:16:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T05:16:40.669Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.181:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.181:8765: connect: connection refused" 2024-03-29T05:16:41.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:16:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T05:16:40.671Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.181:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.181:8765: connect: connection refused" 2024-03-29T05:16:41.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:16:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T05:16:40.671Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.181:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.181:8765: connect: connection refused" 2024-03-29T05:16:41.131 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:16:40 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T05:16:40.672Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.181:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.181:8765: connect: connection refused" 2024-03-29T05:16:41.428 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:16:41 smithi181 podman[163107]: 2024-03-29 05:16:41.215028168 +0000 UTC m=+5.939604137 container cleanup cb0b10fc92dd2dcc778a1b81e1b7bd9da258c43e906f1c84a7c9ebd72d7c2040 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-03-29T05:16:41.428 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:16:41 smithi181 bash[163107]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1 2024-03-29T05:16:41.700 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:16:41 smithi181 podman[163138]: 2024-03-29 05:16:41.424823064 +0000 UTC m=+0.653321270 container remove cb0b10fc92dd2dcc778a1b81e1b7bd9da258c43e906f1c84a7c9ebd72d7c2040 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-03-29T05:16:42.082 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:16:41 smithi181 podman[163234]: 2024-03-29 05:16:41.72699225 +0000 UTC m=+0.017910185 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:16:42.082 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:16:41 smithi181 podman[163234]: 2024-03-29 05:16:41.85105101 +0000 UTC m=+0.141968977 container create be4e039605faae0d5be381e499bd824bdb277ccaa250c0cca01cec755c3e34b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-deactivate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-03-29T05:16:42.082 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:16:42 smithi181 podman[163234]: 2024-03-29 05:16:42.078300402 +0000 UTC m=+0.369218325 container init be4e039605faae0d5be381e499bd824bdb277ccaa250c0cca01cec755c3e34b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T05:16:42.375 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:16:42 smithi181 podman[163234]: 2024-03-29 05:16:42.08362666 +0000 UTC m=+0.374544585 container start be4e039605faae0d5be381e499bd824bdb277ccaa250c0cca01cec755c3e34b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-8da2362) 2024-03-29T05:16:42.375 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:16:42 smithi181 podman[163234]: 2024-03-29 05:16:42.178201346 +0000 UTC m=+0.469119282 container attach be4e039605faae0d5be381e499bd824bdb277ccaa250c0cca01cec755c3e34b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362) 2024-03-29T05:16:42.668 INFO:journalctl@ceph.osd.1.smithi181.stdout:Mar 29 05:16:42 smithi181 podman[163234]: 2024-03-29 05:16:42.371797648 +0000 UTC m=+0.662715573 container died be4e039605faae0d5be381e499bd824bdb277ccaa250c0cca01cec755c3e34b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T05:16:42.997 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.1.service' 2024-03-29T05:16:43.041 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T05:16:43.041 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-03-29T05:16:43.041 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-03-29T05:16:43.041 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.2 2024-03-29T05:16:43.391 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:16:43 smithi181 systemd[1]: Stopping Ceph osd.2 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:16:43.700 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:16:43 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2[126892]: 2024-03-29T05:16:43.386+0000 7fe3a5f44700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T05:16:43.700 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:16:43 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2[126892]: 2024-03-29T05:16:43.386+0000 7fe3a5f44700 -1 osd.2 972 *** Got signal Terminated *** 2024-03-29T05:16:43.700 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:16:43 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2[126892]: 2024-03-29T05:16:43.386+0000 7fe3a5f44700 -1 osd.2 972 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T05:16:44.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:16:44 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:16:44.310Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:16:44.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:16:44 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:16:44.310Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:16:44.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:16:44 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:16:44.311Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:16:49.022 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:16:48 smithi181 podman[163367]: 2024-03-29 05:16:48.741408378 +0000 UTC m=+5.482159430 container died 12c7ca8de2265669d61f2a3cbf5283ed2f12a3e85fa67cea9fae40f5e1b20b5b (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326) 2024-03-29T05:16:49.450 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:16:49 smithi181 podman[163367]: 2024-03-29 05:16:49.200821392 +0000 UTC m=+5.941572440 container cleanup 12c7ca8de2265669d61f2a3cbf5283ed2f12a3e85fa67cea9fae40f5e1b20b5b (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T05:16:49.450 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:16:49 smithi181 bash[163367]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2 2024-03-29T05:16:49.816 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:16:49 smithi181 podman[163389]: 2024-03-29 05:16:49.478212022 +0000 UTC m=+0.738333235 container remove 12c7ca8de2265669d61f2a3cbf5283ed2f12a3e85fa67cea9fae40f5e1b20b5b (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-03-29T05:16:50.125 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:16:49 smithi181 podman[163490]: 2024-03-29 05:16:49.716015316 +0000 UTC m=+0.018099234 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:16:50.126 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:16:49 smithi181 podman[163490]: 2024-03-29 05:16:49.819682999 +0000 UTC m=+0.121766946 container create fa973d207fc06234652795910b55c7a7365fc3b45d0b5e34cf37a7de9def9f03 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, org.label-schema.license=GPLv2) 2024-03-29T05:16:50.126 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:16:50 smithi181 podman[163490]: 2024-03-29 05:16:50.030293488 +0000 UTC m=+0.332377407 container init fa973d207fc06234652795910b55c7a7365fc3b45d0b5e34cf37a7de9def9f03 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-deactivate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-03-29T05:16:50.126 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:16:50 smithi181 podman[163490]: 2024-03-29 05:16:50.03566621 +0000 UTC m=+0.337750115 container start fa973d207fc06234652795910b55c7a7365fc3b45d0b5e34cf37a7de9def9f03 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, GIT_CLEAN=True, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, CEPH_POINT_RELEASE=) 2024-03-29T05:16:50.126 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:16:50 smithi181 podman[163490]: 2024-03-29 05:16:50.121688692 +0000 UTC m=+0.423772610 container attach fa973d207fc06234652795910b55c7a7365fc3b45d0b5e34cf37a7de9def9f03 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-deactivate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-03-29T05:16:50.450 INFO:journalctl@ceph.osd.2.smithi181.stdout:Mar 29 05:16:50 smithi181 podman[163490]: 2024-03-29 05:16:50.328297487 +0000 UTC m=+0.630381400 container died fa973d207fc06234652795910b55c7a7365fc3b45d0b5e34cf37a7de9def9f03 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-2-deactivate, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux ) 2024-03-29T05:16:50.450 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:16:50 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:16:50.159Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:16:50.451 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:16:50 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:16:50.159Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:16:50.451 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:16:50 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:16:50.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:16:50.968 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.2.service' 2024-03-29T05:16:51.007 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T05:16:51.007 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-03-29T05:16:51.008 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-03-29T05:16:51.008 DEBUG:teuthology.orchestra.run.smithi181:> sudo systemctl stop ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.3 2024-03-29T05:16:51.200 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:16:51 smithi181 systemd[1]: Stopping Ceph osd.3 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:16:51.700 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:16:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3[132498]: 2024-03-29T05:16:51.323+0000 7f17bf588700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T05:16:51.700 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:16:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3[132498]: 2024-03-29T05:16:51.323+0000 7f17bf588700 -1 osd.3 972 *** Got signal Terminated *** 2024-03-29T05:16:51.700 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:16:51 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3[132498]: 2024-03-29T05:16:51.323+0000 7f17bf588700 -1 osd.3 972 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T05:16:54.130 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:16:53 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:16:53.745+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:16:54.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:16:54 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:16:54.310Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:16:54.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:16:54 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:16:54.311Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:16:54.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:16:54 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:16:54.312Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:16:55.130 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:16:54 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:16:54.760+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:16:55.700 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:16:55 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3[132498]: 2024-03-29T05:16:55.232+0000 7f17b7b74700 -1 osd.3 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:28.946501+0000 front 2024-03-29T05:16:28.946640+0000 (oldest deadline 2024-03-29T05:16:54.846700+0000) 2024-03-29T05:16:55.795 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:16:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:16:55.487+0000 7f69eaec4700 -1 osd.4 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.404466+0000 front 2024-03-29T05:16:29.404610+0000 (oldest deadline 2024-03-29T05:16:55.304368+0000) 2024-03-29T05:16:56.130 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:16:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:16:55.791+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:16:56.131 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:16:55 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:16:55.826+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:16:56.700 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:16:56 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3[132498]: 2024-03-29T05:16:56.254+0000 7f17b7b74700 -1 osd.3 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:28.946501+0000 front 2024-03-29T05:16:28.946640+0000 (oldest deadline 2024-03-29T05:16:54.846700+0000) 2024-03-29T05:16:56.790 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:16:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:16:56.457+0000 7f69eaec4700 -1 osd.4 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.404466+0000 front 2024-03-29T05:16:29.404610+0000 (oldest deadline 2024-03-29T05:16:55.304368+0000) 2024-03-29T05:16:57.082 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:16:56 smithi181 podman[163620]: 2024-03-29 05:16:56.797641886 +0000 UTC m=+5.576328813 container died 6ac673c0ed7c9b06f751a6b0e5ef3fc822e33217398084d891807d612e7ab377 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-03-29T05:16:57.130 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:16:57 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:16:57.007+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:30.618551+0000 front 2024-03-29T05:16:30.618658+0000 (oldest deadline 2024-03-29T05:16:56.518543+0000) 2024-03-29T05:16:57.131 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:16:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:16:56.828+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:16:57.131 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:16:56 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:16:56.786+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:16:57.450 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:16:57 smithi181 podman[163620]: 2024-03-29 05:16:57.252864785 +0000 UTC m=+6.031551709 container cleanup 6ac673c0ed7c9b06f751a6b0e5ef3fc822e33217398084d891807d612e7ab377 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, ceph=True, org.label-schema.vendor=CentOS) 2024-03-29T05:16:57.450 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:16:57 smithi181 bash[163620]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3 2024-03-29T05:16:57.793 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:16:57 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:16:57.408+0000 7f69eaec4700 -1 osd.4 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.404466+0000 front 2024-03-29T05:16:29.404610+0000 (oldest deadline 2024-03-29T05:16:55.304368+0000) 2024-03-29T05:16:57.850 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:16:57 smithi181 podman[163643]: 2024-03-29 05:16:57.537453779 +0000 UTC m=+0.738580075 container remove 6ac673c0ed7c9b06f751a6b0e5ef3fc822e33217398084d891807d612e7ab377 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T05:16:58.127 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:16:57 smithi181 podman[163740]: 2024-03-29 05:16:57.846447039 +0000 UTC m=+0.103430296 container create a913930f9701dd3dbacb49520c08c4cd989a45b39dcc90125be7d6b397fd6b11 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-03-29T05:16:58.127 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:16:57 smithi181 podman[163740]: 2024-03-29 05:16:57.761304279 +0000 UTC m=+0.018287547 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:16:58.127 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:16:58 smithi181 podman[163740]: 2024-03-29 05:16:58.04043154 +0000 UTC m=+0.297414795 container init a913930f9701dd3dbacb49520c08c4cd989a45b39dcc90125be7d6b397fd6b11 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, ceph=True) 2024-03-29T05:16:58.127 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:16:58 smithi181 podman[163740]: 2024-03-29 05:16:58.045512064 +0000 UTC m=+0.302495329 container start a913930f9701dd3dbacb49520c08c4cd989a45b39dcc90125be7d6b397fd6b11 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2) 2024-03-29T05:16:58.127 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:16:58 smithi181 podman[163740]: 2024-03-29 05:16:58.123513229 +0000 UTC m=+0.380496492 container attach a913930f9701dd3dbacb49520c08c4cd989a45b39dcc90125be7d6b397fd6b11 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T05:16:58.130 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:16:57 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:16:57.987+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:30.618551+0000 front 2024-03-29T05:16:30.618658+0000 (oldest deadline 2024-03-29T05:16:56.518543+0000) 2024-03-29T05:16:58.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:16:57 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:16:57.824+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:16:58.130 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:16:57 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:16:57.789+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:16:58.450 INFO:journalctl@ceph.osd.3.smithi181.stdout:Mar 29 05:16:58 smithi181 podman[163740]: 2024-03-29 05:16:58.329383671 +0000 UTC m=+0.586366917 container died a913930f9701dd3dbacb49520c08c4cd989a45b39dcc90125be7d6b397fd6b11 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362) 2024-03-29T05:16:58.820 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:16:58 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:16:58.405+0000 7f69eaec4700 -1 osd.4 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.404466+0000 front 2024-03-29T05:16:29.404610+0000 (oldest deadline 2024-03-29T05:16:55.304368+0000) 2024-03-29T05:16:59.032 DEBUG:teuthology.orchestra.run.smithi181:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.3.service' 2024-03-29T05:16:59.074 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T05:16:59.074 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-03-29T05:16:59.074 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-03-29T05:16:59.074 DEBUG:teuthology.orchestra.run.smithi187:> sudo systemctl stop ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.4 2024-03-29T05:16:59.106 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:16:58 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:16:58.970+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:30.618551+0000 front 2024-03-29T05:16:30.618658+0000 (oldest deadline 2024-03-29T05:16:56.518543+0000) 2024-03-29T05:16:59.106 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:16:58 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:16:58.859+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:16:59.107 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:16:58 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:16:58.816+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:16:59.367 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:16:59 smithi187 systemd[1]: Stopping Ceph osd.4 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:16:59.630 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:16:59 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:16:59.363+0000 7f69eaec4700 -1 osd.4 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.404466+0000 front 2024-03-29T05:16:29.404610+0000 (oldest deadline 2024-03-29T05:16:55.304368+0000) 2024-03-29T05:16:59.630 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:16:59 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:16:59.565+0000 7f69ee8d0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T05:16:59.631 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:16:59 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:16:59.565+0000 7f69ee8d0700 -1 osd.4 972 *** Got signal Terminated *** 2024-03-29T05:16:59.631 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:16:59 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:16:59.565+0000 7f69ee8d0700 -1 osd.4 972 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T05:17:00.130 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:00 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:00.013+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:30.618551+0000 front 2024-03-29T05:16:30.618658+0000 (oldest deadline 2024-03-29T05:16:56.518543+0000) 2024-03-29T05:17:00.131 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:16:59 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:16:59.898+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:00.131 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:16:59 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:16:59.845+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:00.450 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:00 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:00.159Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:17:00.450 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:00 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:00.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:17:00.450 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:00 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:00.161Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:17:00.828 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:00 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:17:00.378+0000 7f69eaec4700 -1 osd.4 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.404466+0000 front 2024-03-29T05:16:29.404610+0000 (oldest deadline 2024-03-29T05:16:55.304368+0000) 2024-03-29T05:17:01.130 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:01 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:01.053+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:30.618551+0000 front 2024-03-29T05:16:30.618658+0000 (oldest deadline 2024-03-29T05:16:56.518543+0000) 2024-03-29T05:17:01.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:00 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:00.934+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:01.131 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:00 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:00.823+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:01.630 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:01 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:17:01.336+0000 7f69eaec4700 -1 osd.4 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.404466+0000 front 2024-03-29T05:16:29.404610+0000 (oldest deadline 2024-03-29T05:16:55.304368+0000) 2024-03-29T05:17:02.130 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:02 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:02.051+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:30.618551+0000 front 2024-03-29T05:16:30.618658+0000 (oldest deadline 2024-03-29T05:16:56.518543+0000) 2024-03-29T05:17:02.131 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:01 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:01.916+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:02.131 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:01 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:01.916+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:02.131 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:01 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:01.857+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:02.131 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:01 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:01.857+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:02.630 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:02 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:17:02.341+0000 7f69eaec4700 -1 osd.4 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.404466+0000 front 2024-03-29T05:16:29.404610+0000 (oldest deadline 2024-03-29T05:16:55.304368+0000) 2024-03-29T05:17:03.087 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:02 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:02.940+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:03.087 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:02 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:02.940+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:03.087 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:02 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:02.810+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:03.088 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:02 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:02.810+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:03.355 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:03.083+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:30.618551+0000 front 2024-03-29T05:16:30.618658+0000 (oldest deadline 2024-03-29T05:16:56.518543+0000) 2024-03-29T05:17:03.355 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:03.083+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:38.819584+0000 front 2024-03-29T05:16:38.819550+0000 (oldest deadline 2024-03-29T05:17:02.919497+0000) 2024-03-29T05:17:03.630 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:17:03.351+0000 7f69eaec4700 -1 osd.4 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.404466+0000 front 2024-03-29T05:16:29.404610+0000 (oldest deadline 2024-03-29T05:16:55.304368+0000) 2024-03-29T05:17:03.630 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:17:03.351+0000 7f69eaec4700 -1 osd.4 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.005234+0000 front 2024-03-29T05:16:37.005336+0000 (oldest deadline 2024-03-29T05:17:02.905225+0000) 2024-03-29T05:17:04.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:03.897+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:04.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:03.897+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:04.131 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:03.794+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:04.131 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:03 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:03.794+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:04.630 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:17:04.306+0000 7f69eaec4700 -1 osd.4 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.404466+0000 front 2024-03-29T05:16:29.404610+0000 (oldest deadline 2024-03-29T05:16:55.304368+0000) 2024-03-29T05:17:04.630 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4[101604]: 2024-03-29T05:17:04.306+0000 7f69eaec4700 -1 osd.4 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.005234+0000 front 2024-03-29T05:16:37.005336+0000 (oldest deadline 2024-03-29T05:17:02.905225+0000) 2024-03-29T05:17:04.631 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:04.126+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:30.618551+0000 front 2024-03-29T05:16:30.618658+0000 (oldest deadline 2024-03-29T05:16:56.518543+0000) 2024-03-29T05:17:04.631 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:04.126+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:38.819584+0000 front 2024-03-29T05:16:38.819550+0000 (oldest deadline 2024-03-29T05:17:02.919497+0000) 2024-03-29T05:17:04.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:04 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:04.311Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:17:04.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:04 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:04.311Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:17:04.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:04 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:04.312Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:17:05.117 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:04.869+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:05.118 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:04.869+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:05.118 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:04.795+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:05.118 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:04 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:04.795+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:05.380 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:05 smithi187 podman[114534]: 2024-03-29 05:17:05.113600864 +0000 UTC m=+5.845028166 container died 49e5104e4a5d385928a561b1da11f7dae432886672d933441b9e9da4f5cb2716 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-03-29T05:17:05.380 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:05 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:05.137+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:30.618551+0000 front 2024-03-29T05:16:30.618658+0000 (oldest deadline 2024-03-29T05:16:56.518543+0000) 2024-03-29T05:17:05.380 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:05 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:05.137+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:38.819584+0000 front 2024-03-29T05:16:38.819550+0000 (oldest deadline 2024-03-29T05:17:02.919497+0000) 2024-03-29T05:17:05.742 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:05 smithi187 podman[114534]: 2024-03-29 05:17:05.520935662 +0000 UTC m=+6.252362972 container cleanup 49e5104e4a5d385928a561b1da11f7dae432886672d933441b9e9da4f5cb2716 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T05:17:05.742 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:05 smithi187 bash[114534]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4 2024-03-29T05:17:06.079 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:05 smithi187 podman[114556]: 2024-03-29 05:17:05.738879152 +0000 UTC m=+0.622046890 container remove 49e5104e4a5d385928a561b1da11f7dae432886672d933441b9e9da4f5cb2716 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-03-29T05:17:06.079 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:05 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:05.901+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:06.079 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:05 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:05.901+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:06.080 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:05 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:05.778+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:06.080 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:05 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:05.778+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:06.338 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:06 smithi187 podman[114663]: 2024-03-29 05:17:05.978845288 +0000 UTC m=+0.018716750 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T05:17:06.338 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:06 smithi187 podman[114663]: 2024-03-29 05:17:06.098114853 +0000 UTC m=+0.137986306 container create 39ad6e28b0d71b1dce48586304dfb0cd5886c7edd91e11081e4dfa4e250d5b8b (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-deactivate, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T05:17:06.338 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:06 smithi187 podman[114663]: 2024-03-29 05:17:06.333977476 +0000 UTC m=+0.373848943 container init 39ad6e28b0d71b1dce48586304dfb0cd5886c7edd91e11081e4dfa4e250d5b8b (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-03-29T05:17:06.339 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:06 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:06.126+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:30.618551+0000 front 2024-03-29T05:16:30.618658+0000 (oldest deadline 2024-03-29T05:16:56.518543+0000) 2024-03-29T05:17:06.339 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:06 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:06.126+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:38.819584+0000 front 2024-03-29T05:16:38.819550+0000 (oldest deadline 2024-03-29T05:17:02.919497+0000) 2024-03-29T05:17:06.630 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:06 smithi187 podman[114663]: 2024-03-29 05:17:06.34130019 +0000 UTC m=+0.381171643 container start 39ad6e28b0d71b1dce48586304dfb0cd5886c7edd91e11081e4dfa4e250d5b8b (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-03-29T05:17:06.630 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:06 smithi187 podman[114663]: 2024-03-29 05:17:06.450593365 +0000 UTC m=+0.490464818 container attach 39ad6e28b0d71b1dce48586304dfb0cd5886c7edd91e11081e4dfa4e250d5b8b (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-deactivate, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-03-29T05:17:06.952 INFO:journalctl@ceph.osd.4.smithi187.stdout:Mar 29 05:17:06 smithi187 podman[114663]: 2024-03-29 05:17:06.632059807 +0000 UTC m=+0.671931260 container died 39ad6e28b0d71b1dce48586304dfb0cd5886c7edd91e11081e4dfa4e250d5b8b (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-4-deactivate, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-8da2362) 2024-03-29T05:17:06.952 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:06 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:06.824+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:06.953 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:06 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:06.824+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:07.249 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:07 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:07.124+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:30.618551+0000 front 2024-03-29T05:16:30.618658+0000 (oldest deadline 2024-03-29T05:16:56.518543+0000) 2024-03-29T05:17:07.249 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:07 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:07.124+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:38.819584+0000 front 2024-03-29T05:16:38.819550+0000 (oldest deadline 2024-03-29T05:17:02.919497+0000) 2024-03-29T05:17:07.249 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:06 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:06.947+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:07.250 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:06 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:06.947+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:07.275 DEBUG:teuthology.orchestra.run.smithi187:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.4.service' 2024-03-29T05:17:07.358 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T05:17:07.358 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-03-29T05:17:07.358 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-03-29T05:17:07.359 DEBUG:teuthology.orchestra.run.smithi187:> sudo systemctl stop ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.5 2024-03-29T05:17:07.630 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:07 smithi187 systemd[1]: Stopping Ceph osd.5 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:17:07.940 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:07 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:07.779+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:07.940 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:07 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:07.780+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:07.940 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:07 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:07.667+0000 7f67d2b75700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T05:17:07.941 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:07 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:07.667+0000 7f67d2b75700 -1 osd.5 972 *** Got signal Terminated *** 2024-03-29T05:17:07.941 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:07 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:07.667+0000 7f67d2b75700 -1 osd.5 972 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T05:17:08.380 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:07 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:07.935+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:08.380 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:07 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:07.935+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:08.381 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:08.162+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:30.618551+0000 front 2024-03-29T05:16:30.618658+0000 (oldest deadline 2024-03-29T05:16:56.518543+0000) 2024-03-29T05:17:08.381 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:08.162+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:38.819584+0000 front 2024-03-29T05:16:38.819550+0000 (oldest deadline 2024-03-29T05:17:02.919497+0000) 2024-03-29T05:17:09.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:08.897+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:09.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:08.897+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:09.131 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:08.771+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:09.131 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:08 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:08.771+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:09.630 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:09 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:09.171+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:30.618551+0000 front 2024-03-29T05:16:30.618658+0000 (oldest deadline 2024-03-29T05:16:56.518543+0000) 2024-03-29T05:17:09.630 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:09 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:09.171+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:38.819584+0000 front 2024-03-29T05:16:38.819550+0000 (oldest deadline 2024-03-29T05:17:02.919497+0000) 2024-03-29T05:17:10.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:09 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:09.937+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:10.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:09 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:09.937+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:10.131 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:09 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:09.742+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:10.131 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:09 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:09.742+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:10.131 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:09 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:09.742+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:10.450 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:10 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:10.161Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:17:10.450 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:10 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:10.161Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:17:10.451 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:10 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:10.162Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:17:10.630 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:10.183+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:30.618551+0000 front 2024-03-29T05:16:30.618658+0000 (oldest deadline 2024-03-29T05:16:56.518543+0000) 2024-03-29T05:17:10.631 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:10.183+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:38.819584+0000 front 2024-03-29T05:16:38.819550+0000 (oldest deadline 2024-03-29T05:17:02.919497+0000) 2024-03-29T05:17:10.631 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:10.183+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:46.420429+0000 front 2024-03-29T05:16:46.420407+0000 (oldest deadline 2024-03-29T05:17:09.920322+0000) 2024-03-29T05:17:11.130 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:10.699+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:11.130 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:10.699+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:11.131 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:10.699+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:11.131 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:10.892+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:11.131 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:10.892+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:11.131 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:10 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:10.892+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.097869+0000 front 2024-03-29T05:16:47.097882+0000 (oldest deadline 2024-03-29T05:17:09.997725+0000) 2024-03-29T05:17:11.630 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:11.226+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:30.618551+0000 front 2024-03-29T05:16:30.618658+0000 (oldest deadline 2024-03-29T05:16:56.518543+0000) 2024-03-29T05:17:11.631 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:11.226+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:38.819584+0000 front 2024-03-29T05:16:38.819550+0000 (oldest deadline 2024-03-29T05:17:02.919497+0000) 2024-03-29T05:17:11.631 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:11.226+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:46.420429+0000 front 2024-03-29T05:16:46.420407+0000 (oldest deadline 2024-03-29T05:17:09.920322+0000) 2024-03-29T05:17:11.917 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:11.654+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:11.917 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:11.654+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:11.917 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:11.654+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:12.279 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:11.913+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:12.280 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:11.913+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:12.280 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:11 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:11.913+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.097869+0000 front 2024-03-29T05:16:47.097882+0000 (oldest deadline 2024-03-29T05:17:09.997725+0000) 2024-03-29T05:17:12.617 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:12 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:12.275+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:30.618551+0000 front 2024-03-29T05:16:30.618658+0000 (oldest deadline 2024-03-29T05:16:56.518543+0000) 2024-03-29T05:17:12.617 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:12 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:12.275+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:38.819584+0000 front 2024-03-29T05:16:38.819550+0000 (oldest deadline 2024-03-29T05:17:02.919497+0000) 2024-03-29T05:17:12.617 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:12 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5[45519]: 2024-03-29T05:17:12.275+0000 7f67cc16d700 -1 osd.5 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:46.420429+0000 front 2024-03-29T05:16:46.420407+0000 (oldest deadline 2024-03-29T05:17:09.920322+0000) 2024-03-29T05:17:12.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:12 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:12.612+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:12.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:12 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:12.612+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:12.881 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:12 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:12.612+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:13.380 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:13 smithi187 podman[114796]: 2024-03-29 05:17:13.093991352 +0000 UTC m=+5.522346601 container died 9599ff85a9e7b7787c5617e7537e93e6c95f23a8ccc2d75408d04da408859c22 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=quincy-4213e17) 2024-03-29T05:17:13.381 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:12 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:12.909+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:13.381 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:12 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:12.909+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:13.381 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:12 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:12.909+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.097869+0000 front 2024-03-29T05:16:47.097882+0000 (oldest deadline 2024-03-29T05:17:09.997725+0000) 2024-03-29T05:17:13.710 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:13 smithi187 podman[114796]: 2024-03-29 05:17:13.472832872 +0000 UTC m=+5.901188112 container cleanup 9599ff85a9e7b7787c5617e7537e93e6c95f23a8ccc2d75408d04da408859c22 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-03-29T05:17:13.711 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:13 smithi187 bash[114796]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5 2024-03-29T05:17:13.711 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:13 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:13.568+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:13.711 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:13 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:13.568+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:13.711 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:13 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:13.568+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:14.032 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:13 smithi187 podman[114820]: 2024-03-29 05:17:13.70722453 +0000 UTC m=+0.613993847 container remove 9599ff85a9e7b7787c5617e7537e93e6c95f23a8ccc2d75408d04da408859c22 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T05:17:14.032 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:13 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:13.919+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:14.033 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:13 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:13.919+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:14.033 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:13 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:13.919+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.097869+0000 front 2024-03-29T05:16:47.097882+0000 (oldest deadline 2024-03-29T05:17:09.997725+0000) 2024-03-29T05:17:14.339 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:14 smithi187 podman[114919]: 2024-03-29 05:17:13.931530232 +0000 UTC m=+0.018494586 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b 2024-03-29T05:17:14.339 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:14 smithi187 podman[114919]: 2024-03-29 05:17:14.033187781 +0000 UTC m=+0.120152117 container create 3b4b94e64e7dddf4973820b519346434dac218f930542b485f395fb3bcd87f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5-deactivate, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, RELEASE=quincy-4213e17, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326) 2024-03-29T05:17:14.340 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:14 smithi187 podman[114919]: 2024-03-29 05:17:14.235799311 +0000 UTC m=+0.322763660 container init 3b4b94e64e7dddf4973820b519346434dac218f930542b485f395fb3bcd87f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5-deactivate, RELEASE=quincy-4213e17, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326) 2024-03-29T05:17:14.340 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:14 smithi187 podman[114919]: 2024-03-29 05:17:14.241647706 +0000 UTC m=+0.328612042 container start 3b4b94e64e7dddf4973820b519346434dac218f930542b485f395fb3bcd87f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5-deactivate, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-03-29T05:17:14.340 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:14 smithi187 podman[114919]: 2024-03-29 05:17:14.335537957 +0000 UTC m=+0.422502328 container attach 3b4b94e64e7dddf4973820b519346434dac218f930542b485f395fb3bcd87f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5-deactivate, RELEASE=quincy-4213e17, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-03-29T05:17:14.630 INFO:journalctl@ceph.osd.5.smithi187.stdout:Mar 29 05:17:14 smithi187 podman[114919]: 2024-03-29 05:17:14.521161285 +0000 UTC m=+0.608125625 container died 3b4b94e64e7dddf4973820b519346434dac218f930542b485f395fb3bcd87f42 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17) 2024-03-29T05:17:14.631 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:14 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:14.554+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:14.631 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:14 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:14.554+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:14.631 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:14 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:14.554+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:14.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:14 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:14.311Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:17:14.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:14 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:14.312Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:17:14.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:14 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:14.312Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:17:15.159 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:14 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:14.880+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:15.159 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:14 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:14.880+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:15.160 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:14 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:14.880+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.097869+0000 front 2024-03-29T05:16:47.097882+0000 (oldest deadline 2024-03-29T05:17:09.997725+0000) 2024-03-29T05:17:15.182 DEBUG:teuthology.orchestra.run.smithi187:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.5.service' 2024-03-29T05:17:15.223 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T05:17:15.224 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-03-29T05:17:15.225 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-03-29T05:17:15.225 DEBUG:teuthology.orchestra.run.smithi187:> sudo systemctl stop ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.6 2024-03-29T05:17:15.532 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:15 smithi187 systemd[1]: Stopping Ceph osd.6 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:17:15.844 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:15 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:15.527+0000 7f3ff625a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T05:17:15.844 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:15 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:15.527+0000 7f3ff625a700 -1 osd.6 972 *** Got signal Terminated *** 2024-03-29T05:17:15.844 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:15 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:15.527+0000 7f3ff625a700 -1 osd.6 972 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T05:17:15.844 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:15 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:15.555+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:15.845 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:15 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:15.555+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:15.845 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:15 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:15.555+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:16.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:15 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:15.924+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:16.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:15 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:15.924+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:16.131 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:15 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:15.924+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.097869+0000 front 2024-03-29T05:16:47.097882+0000 (oldest deadline 2024-03-29T05:17:09.997725+0000) 2024-03-29T05:17:16.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:16.592+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:16.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:16.592+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:16.881 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:16.592+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:17.380 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:16.903+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:17.380 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:16.903+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:17.380 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:16 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:16.903+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.097869+0000 front 2024-03-29T05:16:47.097882+0000 (oldest deadline 2024-03-29T05:17:09.997725+0000) 2024-03-29T05:17:17.861 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:17 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:17.555+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:17.861 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:17 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:17.555+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:17.862 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:17 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:17.555+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:18.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:17 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:17.857+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:18.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:17 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:17.857+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:18.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:17 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:17.857+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.097869+0000 front 2024-03-29T05:16:47.097882+0000 (oldest deadline 2024-03-29T05:17:09.997725+0000) 2024-03-29T05:17:18.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:17 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:17.857+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6830 osd.3 since back 2024-03-29T05:16:55.298678+0000 front 2024-03-29T05:16:55.298637+0000 (oldest deadline 2024-03-29T05:17:16.998556+0000) 2024-03-29T05:17:18.824 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:18.523+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:18.824 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:18.523+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:18.824 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:18.523+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:19.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:18.820+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:19.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:18.820+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:19.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:18.820+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.097869+0000 front 2024-03-29T05:16:47.097882+0000 (oldest deadline 2024-03-29T05:17:09.997725+0000) 2024-03-29T05:17:19.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:18 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:18.820+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6830 osd.3 since back 2024-03-29T05:16:55.298678+0000 front 2024-03-29T05:16:55.298637+0000 (oldest deadline 2024-03-29T05:17:16.998556+0000) 2024-03-29T05:17:19.817 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:19 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:19.552+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:19.817 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:19 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:19.552+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:19.818 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:19 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:19.552+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:20.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:19 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:19.813+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.595781+0000 front 2024-03-29T05:16:29.595794+0000 (oldest deadline 2024-03-29T05:16:54.895498+0000) 2024-03-29T05:17:20.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:19 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:19.813+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:40.196379+0000 front 2024-03-29T05:16:40.196448+0000 (oldest deadline 2024-03-29T05:17:01.196929+0000) 2024-03-29T05:17:20.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:19 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:19.813+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.097869+0000 front 2024-03-29T05:16:47.097882+0000 (oldest deadline 2024-03-29T05:17:09.997725+0000) 2024-03-29T05:17:20.130 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:19 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6[51927]: 2024-03-29T05:17:19.813+0000 7f3ff1055700 -1 osd.6 972 heartbeat_check: no reply from 172.21.15.181:6830 osd.3 since back 2024-03-29T05:16:55.298678+0000 front 2024-03-29T05:16:55.298637+0000 (oldest deadline 2024-03-29T05:17:16.998556+0000) 2024-03-29T05:17:20.450 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:20.161Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:17:20.450 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:20.161Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:17:20.450 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:20 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:20.162Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:17:20.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:20 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:20.584+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:20.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:20 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:20.584+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:20.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:20 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:20.584+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:20.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:20 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:20.584+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6830 osd.3 since back 2024-03-29T05:16:54.994493+0000 front 2024-03-29T05:16:54.994541+0000 (oldest deadline 2024-03-29T05:17:19.694469+0000) 2024-03-29T05:17:21.229 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:20 smithi187 podman[115048]: 2024-03-29 05:17:20.972824833 +0000 UTC m=+5.538002298 container died c3287470c1faecf91001c3ed16e6f2bd016f35575041c8bca1da444346c74662 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, ceph=True) 2024-03-29T05:17:21.587 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:21 smithi187 podman[115048]: 2024-03-29 05:17:21.332774471 +0000 UTC m=+5.897951937 container cleanup c3287470c1faecf91001c3ed16e6f2bd016f35575041c8bca1da444346c74662 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T05:17:21.588 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:21 smithi187 bash[115048]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6 2024-03-29T05:17:21.881 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:21 smithi187 podman[115073]: 2024-03-29 05:17:21.584057503 +0000 UTC m=+0.607403393 container remove c3287470c1faecf91001c3ed16e6f2bd016f35575041c8bca1da444346c74662 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=quincy-4213e17, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True) 2024-03-29T05:17:21.882 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:21 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:21.614+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:21.882 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:21 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:21.614+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:21.883 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:21 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:21.614+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:21.883 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:21 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:21.614+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6830 osd.3 since back 2024-03-29T05:16:54.994493+0000 front 2024-03-29T05:16:54.994541+0000 (oldest deadline 2024-03-29T05:17:19.694469+0000) 2024-03-29T05:17:22.191 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:21 smithi187 podman[115171]: 2024-03-29 05:17:21.893227527 +0000 UTC m=+0.111738572 container create 89e775c908ae5fdfa03a96d72def693b18d5c459fd12cf675f8420b7a4bd1f1a (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6-deactivate, RELEASE=quincy-4213e17, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-03-29T05:17:22.191 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:21 smithi187 podman[115171]: 2024-03-29 05:17:21.799050144 +0000 UTC m=+0.017561172 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b 2024-03-29T05:17:22.191 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:22 smithi187 podman[115171]: 2024-03-29 05:17:22.095834612 +0000 UTC m=+0.314345622 container init 89e775c908ae5fdfa03a96d72def693b18d5c459fd12cf675f8420b7a4bd1f1a (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6-deactivate, RELEASE=quincy-4213e17, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-03-29T05:17:22.191 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:22 smithi187 podman[115171]: 2024-03-29 05:17:22.101320701 +0000 UTC m=+0.319831700 container start 89e775c908ae5fdfa03a96d72def693b18d5c459fd12cf675f8420b7a4bd1f1a (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, RELEASE=quincy-4213e17, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T05:17:22.191 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:22 smithi187 podman[115171]: 2024-03-29 05:17:22.187218567 +0000 UTC m=+0.405729572 container attach 89e775c908ae5fdfa03a96d72def693b18d5c459fd12cf675f8420b7a4bd1f1a (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-03-29T05:17:22.602 INFO:journalctl@ceph.osd.6.smithi187.stdout:Mar 29 05:17:22 smithi187 podman[115171]: 2024-03-29 05:17:22.376037728 +0000 UTC m=+0.594548735 container died 89e775c908ae5fdfa03a96d72def693b18d5c459fd12cf675f8420b7a4bd1f1a (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-6-deactivate, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-29T05:17:22.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:22.598+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:22.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:22.598+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:22.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:22.598+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:22.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:22 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:22.598+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6830 osd.3 since back 2024-03-29T05:16:54.994493+0000 front 2024-03-29T05:16:54.994541+0000 (oldest deadline 2024-03-29T05:17:19.694469+0000) 2024-03-29T05:17:22.992 DEBUG:teuthology.orchestra.run.smithi187:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.6.service' 2024-03-29T05:17:23.032 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T05:17:23.033 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-03-29T05:17:23.033 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-03-29T05:17:23.033 DEBUG:teuthology.orchestra.run.smithi187:> sudo systemctl stop ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.7 2024-03-29T05:17:23.367 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:23 smithi187 systemd[1]: Stopping Ceph osd.7 for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:17:23.630 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:23 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:23.362+0000 7ff755dc2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T05:17:23.630 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:23 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:23.362+0000 7ff755dc2700 -1 osd.7 972 *** Got signal Terminated *** 2024-03-29T05:17:23.630 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:23 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:23.362+0000 7ff755dc2700 -1 osd.7 972 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T05:17:23.630 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:23 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:23.580+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:23.631 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:23 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:23.580+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:23.631 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:23 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:23.580+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:23.631 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:23 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:23.580+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6830 osd.3 since back 2024-03-29T05:16:54.994493+0000 front 2024-03-29T05:16:54.994541+0000 (oldest deadline 2024-03-29T05:17:19.694469+0000) 2024-03-29T05:17:24.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:24 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:24.312Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:17:24.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:24 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:24.312Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:17:24.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:24 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:24.313Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:17:24.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:24 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:24.551+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:24.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:24 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:24.551+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:24.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:24 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:24.551+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:24.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:24 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:24.551+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6830 osd.3 since back 2024-03-29T05:16:54.994493+0000 front 2024-03-29T05:16:54.994541+0000 (oldest deadline 2024-03-29T05:17:19.694469+0000) 2024-03-29T05:17:25.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:25.536+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:25.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:25.536+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:25.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:25.536+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:25.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:25 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:25.536+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6830 osd.3 since back 2024-03-29T05:16:54.994493+0000 front 2024-03-29T05:16:54.994541+0000 (oldest deadline 2024-03-29T05:17:19.694469+0000) 2024-03-29T05:17:26.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:26 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:26.576+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:26.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:26 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:26.576+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:26.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:26 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:26.576+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:26.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:26 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:26.576+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6830 osd.3 since back 2024-03-29T05:16:54.994493+0000 front 2024-03-29T05:16:54.994541+0000 (oldest deadline 2024-03-29T05:17:19.694469+0000) 2024-03-29T05:17:27.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:27.558+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6806 osd.0 since back 2024-03-29T05:16:29.791769+0000 front 2024-03-29T05:16:29.791900+0000 (oldest deadline 2024-03-29T05:16:53.291748+0000) 2024-03-29T05:17:27.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:27.558+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6814 osd.1 since back 2024-03-29T05:16:37.392667+0000 front 2024-03-29T05:16:37.392754+0000 (oldest deadline 2024-03-29T05:17:01.492562+0000) 2024-03-29T05:17:27.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:27.558+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6822 osd.2 since back 2024-03-29T05:16:47.393583+0000 front 2024-03-29T05:16:47.393560+0000 (oldest deadline 2024-03-29T05:17:09.093478+0000) 2024-03-29T05:17:27.880 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:27 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7[58100]: 2024-03-29T05:17:27.558+0000 7ff7533c2700 -1 osd.7 972 heartbeat_check: no reply from 172.21.15.181:6830 osd.3 since back 2024-03-29T05:16:54.994493+0000 front 2024-03-29T05:16:54.994541+0000 (oldest deadline 2024-03-29T05:17:19.694469+0000) 2024-03-29T05:17:29.039 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:28 smithi187 podman[115302]: 2024-03-29 05:17:28.767717399 +0000 UTC m=+5.521391045 container died 89fb8ccf8705232455ecf9d238b2766f48499f316870b3189a28cb7107dbb51b (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-03-29T05:17:29.289 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:29 smithi187 podman[115302]: 2024-03-29 05:17:29.109086302 +0000 UTC m=+5.862759946 container cleanup 89fb8ccf8705232455ecf9d238b2766f48499f316870b3189a28cb7107dbb51b (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-03-29T05:17:29.289 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:29 smithi187 bash[115302]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7 2024-03-29T05:17:29.630 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:29 smithi187 podman[115328]: 2024-03-29 05:17:29.285382174 +0000 UTC m=+0.513666460 container remove 89fb8ccf8705232455ecf9d238b2766f48499f316870b3189a28cb7107dbb51b (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-4213e17, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-03-29T05:17:30.008 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:29 smithi187 podman[115422]: 2024-03-29 05:17:29.694609819 +0000 UTC m=+0.112904086 container create 37be96e2b98aec8f375c4112e1c2dee4969ad83a43bfb11144a8ae14876a7b03 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-03-29T05:17:30.009 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:29 smithi187 podman[115422]: 2024-03-29 05:17:29.599435114 +0000 UTC m=+0.017729396 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b 2024-03-29T05:17:30.009 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:29 smithi187 podman[115422]: 2024-03-29 05:17:29.913642054 +0000 UTC m=+0.331936329 container init 37be96e2b98aec8f375c4112e1c2dee4969ad83a43bfb11144a8ae14876a7b03 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7-deactivate, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326) 2024-03-29T05:17:30.009 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:29 smithi187 podman[115422]: 2024-03-29 05:17:29.919382277 +0000 UTC m=+0.337676541 container start 37be96e2b98aec8f375c4112e1c2dee4969ad83a43bfb11144a8ae14876a7b03 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7-deactivate, RELEASE=quincy-4213e17, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326) 2024-03-29T05:17:30.009 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:30 smithi187 podman[115422]: 2024-03-29 05:17:30.005101632 +0000 UTC m=+0.423395905 container attach 37be96e2b98aec8f375c4112e1c2dee4969ad83a43bfb11144a8ae14876a7b03 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T05:17:30.380 INFO:journalctl@ceph.osd.7.smithi187.stdout:Mar 29 05:17:30 smithi187 podman[115422]: 2024-03-29 05:17:30.201933354 +0000 UTC m=+0.620227629 container died 37be96e2b98aec8f375c4112e1c2dee4969ad83a43bfb11144a8ae14876a7b03 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-osd-7-deactivate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T05:17:30.450 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:30.162Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:17:30.450 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:30.162Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi181.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.181:8443: connect: connection refused" 2024-03-29T05:17:30.450 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:30 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:30.163Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi187.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.187:8443: connect: connection refused" 2024-03-29T05:17:30.833 DEBUG:teuthology.orchestra.run.smithi187:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@osd.7.service' 2024-03-29T05:17:30.873 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T05:17:30.874 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-03-29T05:17:30.874 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-03-29T05:17:30.874 DEBUG:teuthology.orchestra.run.smithi187:> sudo systemctl stop ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@prometheus.a 2024-03-29T05:17:31.130 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:17:30 smithi187 systemd[1]: Stopping Ceph prometheus.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:17:31.524 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:17:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T05:17:31.222Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-29T05:17:31.524 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:17:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T05:17:31.222Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-29T05:17:31.524 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:17:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T05:17:31.222Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-29T05:17:31.525 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:17:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T05:17:31.222Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-29T05:17:31.525 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:17:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T05:17:31.222Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-29T05:17:31.525 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:17:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T05:17:31.222Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-29T05:17:31.525 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:17:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T05:17:31.222Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-29T05:17:31.525 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:17:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T05:17:31.222Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-29T05:17:31.525 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:17:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T05:17:31.222Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-29T05:17:31.525 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:17:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T05:17:31.223Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-29T05:17:31.526 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:17:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T05:17:31.223Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-29T05:17:31.526 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:17:31 smithi187 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a[82984]: ts=2024-03-29T05:17:31.223Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-29T05:17:31.526 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:17:31 smithi187 podman[115556]: 2024-03-29 05:17:31.233022922 +0000 UTC m=+0.145295380 container died 882b8922d74b4354de1ae38e9c7891149e96c0d90602a271459f5d44cd7b5145 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T05:17:31.873 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:17:31 smithi187 podman[115556]: 2024-03-29 05:17:31.636031025 +0000 UTC m=+0.548303466 container cleanup 882b8922d74b4354de1ae38e9c7891149e96c0d90602a271459f5d44cd7b5145 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T05:17:31.874 INFO:journalctl@ceph.prometheus.a.smithi187.stdout:Mar 29 05:17:31 smithi187 bash[115556]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-prometheus-a 2024-03-29T05:17:32.003 DEBUG:teuthology.orchestra.run.smithi187:> sudo pkill -f 'journalctl -f -n 0 -u ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@prometheus.a.service' 2024-03-29T05:17:32.038 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T05:17:32.039 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-03-29T05:17:32.039 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 --force --keep-logs 2024-03-29T05:17:33.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:33 smithi181 systemd[1]: Stopping Ceph alertmanager.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:17:34.082 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:33 smithi181 ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a[91138]: ts=2024-03-29T05:17:33.779Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-03-29T05:17:34.082 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:33 smithi181 podman[164172]: 2024-03-29 05:17:33.820701177 +0000 UTC m=+0.169701240 container died 69e0ad67b57f879ac04b924eef88d502c74d8c54238bf566fc0aa4642ebf3fa2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T05:17:34.449 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:34 smithi181 podman[164172]: 2024-03-29 05:17:34.252923265 +0000 UTC m=+0.601923370 container cleanup 69e0ad67b57f879ac04b924eef88d502c74d8c54238bf566fc0aa4642ebf3fa2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T05:17:34.449 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:34 smithi181 bash[164172]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a 2024-03-29T05:17:34.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:34 smithi181 podman[164192]: 2024-03-29 05:17:34.445701434 +0000 UTC m=+0.622426344 container remove 69e0ad67b57f879ac04b924eef88d502c74d8c54238bf566fc0aa4642ebf3fa2 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T05:17:34.700 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:34 smithi181 podman[164192]: 2024-03-29 05:17:34.529417926 +0000 UTC m=+0.706142790 volume remove f34a0c8bd3e80a46f5462c7d67e3353f87012b10067e4c3a01d363dcc0f0dab6 2024-03-29T05:17:35.200 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 05:17:34 smithi181 systemd[1]: Stopping Ceph node-exporter.a for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:17:35.201 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:34 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-03-29T05:17:35.201 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:34 smithi181 systemd[1]: Stopped Ceph alertmanager.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T05:17:35.201 INFO:journalctl@ceph.alertmanager.a.smithi181.stdout:Mar 29 05:17:34 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@alertmanager.a.service: Consumed 2.635s CPU time. 2024-03-29T05:17:35.502 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 05:17:35 smithi181 podman[164304]: 2024-03-29 05:17:35.228597077 +0000 UTC m=+0.105194814 container died 4029d029600b7db832598baa93643bf1952b1b4c8ba1dc65a966a1223b6924aa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T05:17:35.861 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 05:17:35 smithi181 podman[164304]: 2024-03-29 05:17:35.598624942 +0000 UTC m=+0.475222668 container cleanup 4029d029600b7db832598baa93643bf1952b1b4c8ba1dc65a966a1223b6924aa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T05:17:35.861 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 05:17:35 smithi181 bash[164304]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a 2024-03-29T05:17:36.188 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 05:17:35 smithi181 podman[164323]: 2024-03-29 05:17:35.857824058 +0000 UTC m=+0.633957428 container remove 4029d029600b7db832598baa93643bf1952b1b4c8ba1dc65a966a1223b6924aa (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T05:17:36.189 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 05:17:35 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-03-29T05:17:36.189 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 05:17:36 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@node-exporter.a.service: Failed with result 'exit-code'. 2024-03-29T05:17:36.189 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 05:17:36 smithi181 systemd[1]: Stopped Ceph node-exporter.a for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T05:17:36.189 INFO:journalctl@ceph.node-exporter.a.smithi181.stdout:Mar 29 05:17:36 smithi181 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@node-exporter.a.service: Consumed 3.355s CPU time. 2024-03-29T05:17:46.039 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 --force --keep-logs 2024-03-29T05:17:48.031 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 05:17:47 smithi187 systemd[1]: Stopping Ceph node-exporter.b for dfcecab8-ed86-11ee-95d0-87774f69a715... 2024-03-29T05:17:48.308 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 05:17:48 smithi187 podman[116024]: 2024-03-29 05:17:48.027241619 +0000 UTC m=+0.112111243 container died 3894f6e97d55065f63c4fd150cff365ad2e04d6bb2c387aefe5ffa4ca019b59e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T05:17:48.309 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 05:17:48 smithi187 podman[116024]: 2024-03-29 05:17:48.305170408 +0000 UTC m=+0.390040032 container cleanup 3894f6e97d55065f63c4fd150cff365ad2e04d6bb2c387aefe5ffa4ca019b59e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T05:17:48.575 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 05:17:48 smithi187 bash[116024]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b 2024-03-29T05:17:48.880 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 05:17:48 smithi187 podman[116043]: 2024-03-29 05:17:48.572625139 +0000 UTC m=+0.546024441 container remove 3894f6e97d55065f63c4fd150cff365ad2e04d6bb2c387aefe5ffa4ca019b59e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-dfcecab8-ed86-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T05:17:48.880 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 05:17:48 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-03-29T05:17:48.880 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 05:17:48 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@node-exporter.b.service: Failed with result 'exit-code'. 2024-03-29T05:17:48.880 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 05:17:48 smithi187 systemd[1]: Stopped Ceph node-exporter.b for dfcecab8-ed86-11ee-95d0-87774f69a715. 2024-03-29T05:17:48.880 INFO:journalctl@ceph.node-exporter.b.smithi187.stdout:Mar 29 05:17:48 smithi187 systemd[1]: ceph-dfcecab8-ed86-11ee-95d0-87774f69a715@node-exporter.b.service: Consumed 3.309s CPU time. 2024-03-29T05:17:54.549 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-29T05:17:54.584 DEBUG:teuthology.orchestra.run.smithi187:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-29T05:17:54.614 INFO:tasks.cephadm:Archiving crash dumps... 2024-03-29T05:17:54.617 DEBUG:teuthology.misc:Transferring archived files from smithi181:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/crash to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629121/remote/smithi181/crash 2024-03-29T05:17:54.617 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/crash -- . 2024-03-29T05:17:54.675 INFO:teuthology.orchestra.run.smithi181.stderr:tar: /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/crash: Cannot open: No such file or directory 2024-03-29T05:17:54.675 INFO:teuthology.orchestra.run.smithi181.stderr:tar: Error is not recoverable: exiting now 2024-03-29T05:17:54.677 DEBUG:teuthology.misc:Transferring archived files from smithi187:/var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/crash to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629121/remote/smithi187/crash 2024-03-29T05:17:54.678 DEBUG:teuthology.orchestra.run.smithi187:> sudo tar c -f - -C /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/crash -- . 2024-03-29T05:17:54.720 INFO:teuthology.orchestra.run.smithi187.stderr:tar: /var/lib/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/crash: Cannot open: No such file or directory 2024-03-29T05:17:54.720 INFO:teuthology.orchestra.run.smithi187.stderr:tar: Error is not recoverable: exiting now 2024-03-29T05:17:54.721 INFO:tasks.cephadm:Checking cluster log for badness... 2024-03-29T05:17:54.722 DEBUG:teuthology.orchestra.run.smithi181:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-29T05:17:54.752 INFO:teuthology.orchestra.run.smithi181.stdout:1711687840.0428998 mon.a (mon.0) 685 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T05:17:54.754 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-03-29T05:17:54.754 INFO:tasks.cephadm:Compressing logs... 2024-03-29T05:17:54.754 DEBUG:teuthology.orchestra.run.smithi181:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-29T05:17:54.796 DEBUG:teuthology.orchestra.run.smithi187:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-29T05:17:54.823 INFO:teuthology.orchestra.run.smithi181.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-29T05:17:54.823 INFO:teuthology.orchestra.run.smithi181.stderr:: No such file or directory 2024-03-29T05:17:54.823 INFO:teuthology.orchestra.run.smithi181.stderr:gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mon.a.log 2024-03-29T05:17:54.823 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.audit.log 2024-03-29T05:17:54.824 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.1.log 2024-03-29T05:17:54.824 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.3.log 2024-03-29T05:17:54.833 INFO:teuthology.orchestra.run.smithi181.stderr:gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mds.foo.smithi181.pflrwm.log/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.1.log: 2024-03-29T05:17:54.834 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mgr.y.log 2024-03-29T05:17:54.834 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mds.foo.smithi181.pflrwm.log: gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.0.log 2024-03-29T05:17:54.834 INFO:teuthology.orchestra.run.smithi181.stderr: 81.5%/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mgr.y.log: -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mds.foo.smithi181.pflrwm.log.gz 2024-03-29T05:17:54.834 INFO:teuthology.orchestra.run.smithi181.stderr:gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.cephadm.log 2024-03-29T05:17:54.834 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.log 2024-03-29T05:17:54.834 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.cephadm.log: 90.0% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.cephadm.log.gz 2024-03-29T05:17:54.834 INFO:teuthology.orchestra.run.smithi181.stderr:gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.2.log 2024-03-29T05:17:54.834 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.log: 93.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-29T05:17:54.835 INFO:teuthology.orchestra.run.smithi181.stderr:gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-volume.log 2024-03-29T05:17:54.835 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.2.log: 93.8% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.audit.log.gz 2024-03-29T05:17:54.835 INFO:teuthology.orchestra.run.smithi187.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-29T05:17:54.835 INFO:teuthology.orchestra.run.smithi187.stderr:: No such file or directory 2024-03-29T05:17:54.835 INFO:teuthology.orchestra.run.smithi187.stderr:gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.6.log 2024-03-29T05:17:54.835 INFO:teuthology.orchestra.run.smithi187.stderr:gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.audit.log 2024-03-29T05:17:54.835 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.4.log 2024-03-29T05:17:54.836 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.6.log: /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.7.log 2024-03-29T05:17:54.836 INFO:teuthology.orchestra.run.smithi187.stderr:gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.5.log 2024-03-29T05:17:54.836 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.4.log: /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mds.foo.smithi187.vcsnqx.log 2024-03-29T05:17:54.836 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mgr.x.log 2024-03-29T05:17:54.836 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mds.foo.smithi187.vcsnqx.log: gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.cephadm.log 62.3% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mds.foo.smithi187.vcsnqx.log.gz 2024-03-29T05:17:54.836 INFO:teuthology.orchestra.run.smithi187.stderr: 2024-03-29T05:17:54.836 INFO:teuthology.orchestra.run.smithi187.stderr:gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.log 2024-03-29T05:17:54.837 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.cephadm.log: /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mgr.x.log: 83.2% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.cephadm.log.gz 2024-03-29T05:17:54.837 INFO:teuthology.orchestra.run.smithi187.stderr: 90.0% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.audit.log.gz 2024-03-29T05:17:54.837 INFO:teuthology.orchestra.run.smithi187.stderr:gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-volume.log 2024-03-29T05:17:54.837 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.log: gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mon.b.log 2024-03-29T05:17:54.838 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-volume.log: 91.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-29T05:17:54.839 INFO:teuthology.orchestra.run.smithi187.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mon.b.log: 87.5% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.log.gz 2024-03-29T05:17:54.841 INFO:teuthology.orchestra.run.smithi181.stderr:gzip -5 --verbose -- /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mon.c.log 2024-03-29T05:17:54.844 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-volume.log: 92.5% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph.log.gz 2024-03-29T05:17:54.847 INFO:teuthology.orchestra.run.smithi187.stderr: 91.2% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-volume.log.gz 2024-03-29T05:17:54.862 INFO:teuthology.orchestra.run.smithi181.stderr:/var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mon.c.log: 91.0% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-volume.log.gz 2024-03-29T05:17:54.873 INFO:teuthology.orchestra.run.smithi187.stderr: 89.7% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mgr.x.log.gz 2024-03-29T05:17:55.299 INFO:teuthology.orchestra.run.smithi181.stderr: 89.2% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mgr.y.log.gz 2024-03-29T05:17:55.639 INFO:teuthology.orchestra.run.smithi187.stderr: 90.5% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mon.b.log.gz 2024-03-29T05:17:56.103 INFO:teuthology.orchestra.run.smithi181.stderr: 90.4% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mon.c.log.gz 2024-03-29T05:17:57.231 INFO:teuthology.orchestra.run.smithi181.stderr: 90.7% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-mon.a.log.gz 2024-03-29T05:18:01.857 INFO:teuthology.orchestra.run.smithi187.stderr: 93.7% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.5.log.gz 2024-03-29T05:18:02.252 INFO:teuthology.orchestra.run.smithi187.stderr: 94.1% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.7.log.gz 2024-03-29T05:18:02.927 INFO:teuthology.orchestra.run.smithi181.stderr: 94.6% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.2.log.gz 2024-03-29T05:18:03.231 INFO:teuthology.orchestra.run.smithi181.stderr: 94.3% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.1.log.gz 2024-03-29T05:18:03.664 INFO:teuthology.orchestra.run.smithi181.stderr: 94.4% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.0.log.gz 2024-03-29T05:18:04.083 INFO:teuthology.orchestra.run.smithi187.stderr: 94.5% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.4.log.gz 2024-03-29T05:18:04.453 INFO:teuthology.orchestra.run.smithi181.stderr: 95.0% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.3.log.gz 2024-03-29T05:18:04.455 INFO:teuthology.orchestra.run.smithi181.stderr: 2024-03-29T05:18:04.455 INFO:teuthology.orchestra.run.smithi181.stderr:real 0m9.646s 2024-03-29T05:18:04.455 INFO:teuthology.orchestra.run.smithi181.stderr:user 0m37.646s 2024-03-29T05:18:04.455 INFO:teuthology.orchestra.run.smithi181.stderr:sys 0m1.351s 2024-03-29T05:18:04.589 INFO:teuthology.orchestra.run.smithi187.stderr: 94.3% -- replaced with /var/log/ceph/dfcecab8-ed86-11ee-95d0-87774f69a715/ceph-osd.6.log.gz 2024-03-29T05:18:04.590 INFO:teuthology.orchestra.run.smithi187.stderr: 2024-03-29T05:18:04.591 INFO:teuthology.orchestra.run.smithi187.stderr:real 0m9.780s 2024-03-29T05:18:04.591 INFO:teuthology.orchestra.run.smithi187.stderr:user 0m33.089s 2024-03-29T05:18:04.591 INFO:teuthology.orchestra.run.smithi187.stderr:sys 0m1.256s 2024-03-29T05:18:04.591 INFO:tasks.cephadm:Archiving logs... 2024-03-29T05:18:04.591 DEBUG:teuthology.misc:Transferring archived files from smithi181:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629121/remote/smithi181/log 2024-03-29T05:18:04.592 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-29T05:18:09.501 DEBUG:teuthology.misc:Transferring archived files from smithi187:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629121/remote/smithi187/log 2024-03-29T05:18:09.502 DEBUG:teuthology.orchestra.run.smithi187:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-29T05:18:14.147 INFO:tasks.cephadm:Removing cluster... 2024-03-29T05:18:14.147 DEBUG:teuthology.orchestra.run.smithi181:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 --force 2024-03-29T05:18:14.716 DEBUG:teuthology.orchestra.run.smithi187:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid dfcecab8-ed86-11ee-95d0-87774f69a715 --force 2024-03-29T05:18:15.235 INFO:tasks.cephadm:Removing cephadm ... 2024-03-29T05:18:15.235 DEBUG:teuthology.orchestra.run.smithi181:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-29T05:18:15.251 DEBUG:teuthology.orchestra.run.smithi187:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-29T05:18:15.265 INFO:tasks.cephadm:Teardown complete 2024-03-29T05:18:15.266 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-03-29T05:18:15.278 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-03-29T05:18:15.278 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-03-29T05:18:15.293 DEBUG:teuthology.orchestra.run.smithi187:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-03-29T05:18:15.384 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-03-29T05:18:15.393 INFO:teuthology.task.clock:Checking final clock skew... 2024-03-29T05:18:15.394 DEBUG:teuthology.orchestra.run.smithi181:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-29T05:18:15.398 DEBUG:teuthology.orchestra.run.smithi187:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-29T05:18:15.407 INFO:teuthology.orchestra.run.smithi181.stderr:bash: line 1: ntpq: command not found 2024-03-29T05:18:15.411 INFO:teuthology.orchestra.run.smithi181.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-29T05:18:15.412 INFO:teuthology.orchestra.run.smithi181.stdout:=============================================================================== 2024-03-29T05:18:15.412 INFO:teuthology.orchestra.run.smithi181.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 57 -303us[ -303us] +/- 74ms 2024-03-29T05:18:15.412 INFO:teuthology.orchestra.run.smithi181.stdout:^* hv02.front.sepia.ceph.com 3 6 377 56 -1662us[-1662us] +/- 53ms 2024-03-29T05:18:15.412 INFO:teuthology.orchestra.run.smithi181.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 59 +2612us[+2612us] +/- 78ms 2024-03-29T05:18:15.412 INFO:teuthology.orchestra.run.smithi181.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T05:18:15.413 INFO:teuthology.orchestra.run.smithi187.stderr:bash: line 1: ntpq: command not found 2024-03-29T05:18:15.416 INFO:teuthology.orchestra.run.smithi187.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-29T05:18:15.416 INFO:teuthology.orchestra.run.smithi187.stdout:=============================================================================== 2024-03-29T05:18:15.416 INFO:teuthology.orchestra.run.smithi187.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 59 -303us[ -303us] +/- 74ms 2024-03-29T05:18:15.416 INFO:teuthology.orchestra.run.smithi187.stdout:^* hv02.front.sepia.ceph.com 3 6 377 56 -1655us[-1654us] +/- 53ms 2024-03-29T05:18:15.416 INFO:teuthology.orchestra.run.smithi187.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 59 +2598us[+2599us] +/- 78ms 2024-03-29T05:18:15.416 INFO:teuthology.orchestra.run.smithi187.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T05:18:15.417 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-03-29T05:18:15.426 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-03-29T05:18:15.427 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-03-29T05:18:15.436 DEBUG:teuthology.orchestra.run.smithi181:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-29T05:18:15.549 DEBUG:teuthology.orchestra.run.smithi187:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-29T05:18:15.629 DEBUG:teuthology.orchestra.run.smithi181:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-03-29T05:18:15.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T05:18:15.661 DEBUG:teuthology.orchestra.run.smithi187:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-03-29T05:18:15.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T05:18:15.692 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-03-29T05:18:15.701 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-03-29T05:18:15.711 INFO:teuthology.task.internal:Duration was 3162.068759 seconds 2024-03-29T05:18:15.711 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-03-29T05:18:15.720 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-03-29T05:18:15.721 DEBUG:teuthology.orchestra.run.smithi181:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-29T05:18:15.723 DEBUG:teuthology.orchestra.run.smithi187:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-29T05:18:15.764 INFO:teuthology.orchestra.run.smithi181.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-29T05:18:15.774 INFO:teuthology.orchestra.run.smithi187.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-29T05:18:16.235 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-03-29T05:18:16.235 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi181.front.sepia.ceph.com 2024-03-29T05:18:16.235 DEBUG:teuthology.orchestra.run.smithi181:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-29T05:18:16.294 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi187.front.sepia.ceph.com 2024-03-29T05:18:16.295 DEBUG:teuthology.orchestra.run.smithi187:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-29T05:18:16.324 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-03-29T05:18:16.324 DEBUG:teuthology.orchestra.run.smithi181:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-29T05:18:16.337 DEBUG:teuthology.orchestra.run.smithi187:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-29T05:18:16.392 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-03-29T05:18:16.392 DEBUG:teuthology.orchestra.run.smithi181:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-29T05:18:16.405 DEBUG:teuthology.orchestra.run.smithi187:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-29T05:18:16.929 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-03-29T05:18:16.940 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-03-29T05:18:16.940 DEBUG:teuthology.orchestra.run.smithi181:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-29T05:18:16.966 DEBUG:teuthology.orchestra.run.smithi187:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-29T05:18:16.993 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-03-29T05:18:17.004 DEBUG:teuthology.orchestra.run.smithi181:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-29T05:18:17.009 DEBUG:teuthology.orchestra.run.smithi187:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-29T05:18:17.032 INFO:teuthology.orchestra.run.smithi181.stdout:kernel.core_pattern = core 2024-03-29T05:18:17.059 INFO:teuthology.orchestra.run.smithi187.stdout:kernel.core_pattern = core 2024-03-29T05:18:17.091 DEBUG:teuthology.orchestra.run.smithi181:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-29T05:18:17.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T05:18:17.105 DEBUG:teuthology.orchestra.run.smithi187:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-29T05:18:17.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T05:18:17.145 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-03-29T05:18:17.155 INFO:teuthology.task.internal:Transferring archived files... 2024-03-29T05:18:17.155 DEBUG:teuthology.misc:Transferring archived files from smithi181:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629121/remote/smithi181 2024-03-29T05:18:17.156 DEBUG:teuthology.orchestra.run.smithi181:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-29T05:18:17.207 DEBUG:teuthology.misc:Transferring archived files from smithi187:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629121/remote/smithi187 2024-03-29T05:18:17.207 DEBUG:teuthology.orchestra.run.smithi187:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-29T05:18:17.247 INFO:teuthology.task.internal:Removing archive directory... 2024-03-29T05:18:17.248 DEBUG:teuthology.orchestra.run.smithi181:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-29T05:18:17.250 DEBUG:teuthology.orchestra.run.smithi187:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-29T05:18:17.295 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-03-29T05:18:17.307 INFO:teuthology.task.internal:Not uploading archives. 2024-03-29T05:18:17.308 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-03-29T05:18:17.317 INFO:teuthology.task.internal:Tidying up after the test... 2024-03-29T05:18:17.318 DEBUG:teuthology.orchestra.run.smithi181:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-29T05:18:17.320 DEBUG:teuthology.orchestra.run.smithi187:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-29T05:18:17.332 INFO:teuthology.orchestra.run.smithi181.stdout: 266042 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 29 05:18 /home/ubuntu/cephtest 2024-03-29T05:18:17.350 INFO:teuthology.orchestra.run.smithi187.stdout: 265305 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 29 05:18 /home/ubuntu/cephtest 2024-03-29T05:18:17.351 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-03-29T05:18:17.361 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-03-29T05:18:17.380 INFO:teuthology.nuke:Checking targets against current locks 2024-03-29T05:18:17.407 DEBUG:teuthology.nuke:shortname: smithi181 2024-03-29T05:18:17.407 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-29T05:18:17.425 DEBUG:teuthology.nuke:shortname: smithi187 2024-03-29T05:18:17.425 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-29T05:18:17.430 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi181.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629121', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-29 04:17:20.549232', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIWTb+YgcVQFjXL09OeTN9p0qteWDa9xqwpXbDR6q9VTOOCcGqBY+i/8gpy2eVi44Oz4hh1r0/E/rXwvNwKwlFk='} 2024-03-29T05:18:17.443 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi187.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629121', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-29 04:17:20.550664', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOHJtD/ZBPM72nmXa7NPgOaRpmzsm0gr+sOnHoP1jfpVVtEG400ZEHhZwb9mq4m1hbENWj882yRlaoJQfzDyA9k='} 2024-03-29T05:18:17.479 INFO:teuthology.orchestra.console.smithi181:Power off 2024-03-29T05:18:17.479 DEBUG:teuthology.orchestra.console.smithi181:pexpect command: ipmitool -H smithi181.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-29T05:18:17.497 INFO:teuthology.orchestra.console.smithi187:Power off 2024-03-29T05:18:17.498 DEBUG:teuthology.orchestra.console.smithi187:pexpect command: ipmitool -H smithi187.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-29T05:18:17.509 DEBUG:teuthology.orchestra.console.smithi181:power off output: Chassis Power Control: Down/Off 2024-03-29T05:18:17.510 DEBUG:teuthology.orchestra.console.smithi181:pexpect command: ipmitool -H smithi181.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T05:18:17.524 DEBUG:teuthology.orchestra.console.smithi187:power off output: Chassis Power Control: Down/Off 2024-03-29T05:18:17.524 DEBUG:teuthology.orchestra.console.smithi187:pexpect command: ipmitool -H smithi187.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T05:18:17.537 DEBUG:teuthology.orchestra.console.smithi181:check power output: Chassis Power is on 2024-03-29T05:18:17.558 DEBUG:teuthology.orchestra.console.smithi187:check power output: Chassis Power is on 2024-03-29T05:18:21.538 DEBUG:teuthology.orchestra.console.smithi181:pexpect command: ipmitool -H smithi181.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T05:18:21.559 DEBUG:teuthology.orchestra.console.smithi187:pexpect command: ipmitool -H smithi187.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T05:18:21.650 DEBUG:teuthology.orchestra.console.smithi181:check power output: Chassis Power is on 2024-03-29T05:18:21.670 DEBUG:teuthology.orchestra.console.smithi187:check power output: Chassis Power is on 2024-03-29T05:18:25.654 DEBUG:teuthology.orchestra.console.smithi181:pexpect command: ipmitool -H smithi181.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T05:18:25.671 DEBUG:teuthology.orchestra.console.smithi187:pexpect command: ipmitool -H smithi187.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T05:18:25.766 DEBUG:teuthology.orchestra.console.smithi181:check power output: Chassis Power is off 2024-03-29T05:18:25.784 DEBUG:teuthology.orchestra.console.smithi187:check power output: Chassis Power is off 2024-03-29T05:18:25.867 INFO:teuthology.orchestra.console.smithi181:Power off completed 2024-03-29T05:18:25.885 INFO:teuthology.orchestra.console.smithi187:Power off completed 2024-03-29T05:18:25.990 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} duration: 3162.0687594413757 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=6ef80694eb64479cb4f9f341f9fbf018 status: fail success: false 2024-03-29T05:18:25.990 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-29T05:18:26.072 INFO:teuthology.run:FAIL